diff --git a/.cargo/audit.toml b/.cargo/audit.toml index 4dddb2652..1be28fdc8 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -5,6 +5,7 @@ ignore = [ "RUSTSEC-2026-0194", # quick-xml via pprof/inferno (private profiling-experiments feature) "RUSTSEC-2026-0195", # quick-xml via pprof/inferno (private profiling-experiments feature) "RUSTSEC-2025-0057", # fxhash via monoio (private io-experiments feature) + "RUSTSEC-2026-0441", # pin-utils via loom's futures feature (Loom dev-dependency only) ] informational_warnings = ["unmaintained", "unsound"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 111ef2fa8..b1f409e2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: run: cargo test --locked --verbose - name: Run capability examples run: | - for example in execution_groups execution_profiles supervision_recovery supervision_tree dedicated_thread runtime_isolation byte_budget_pipeline timeout_outcomes graceful_drain protocol_timeouts streams_and_recipients registrations_and_fakes native_lifecycle coordinated_shutdown test_kit; do + for example in execution_groups execution_profiles supervision_recovery supervision_tree dedicated_thread runtime_isolation byte_budget_pipeline timeout_outcomes graceful_drain protocol_timeouts streams_and_recipients registrations_and_fakes native_lifecycle coordinated_shutdown test_kit exit_signals; do timeout 60s cargo run --locked -p spawned-capability-examples --bin "$example" done timeout 60s cargo run --locked -p spawned-capability-examples --features metrics --bin framework_metrics diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml index 27e0d1374..aab77a658 100644 --- a/.github/workflows/verification.yml +++ b/.github/workflows/verification.yml @@ -62,7 +62,7 @@ jobs: - name: Record supervisor lifecycle traces env: LIFECYCLE_TRACE_DIR: ${{ github.workspace }}/lifecycle-traces - run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace + run: cargo test --locked -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace - name: Validate traces and negative controls run: nix run .#check-tla -- --traces-only lifecycle-traces --negative-controls # Which model actions, projected states and transitions the recorded @@ -172,6 +172,10 @@ jobs: env: RUSTFLAGS: --cfg spawned_scheduler_loom run: cargo test --locked -p spawned-concurrency --no-default-features --test mailbox_admission_loom + - name: Model-check the reply slot, job core, link table and monitor fence + env: + RUSTFLAGS: --cfg spawned_scheduler_loom + run: cargo test --locked --release -p spawned-concurrency --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom --test link_fence_loom # Data races in the runtime and its dependencies. The crates forbid # unsafe code, so ThreadSanitizer is the sanitizer that matters here. diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f39dfc8..a9410fdda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,306 @@ ## Unreleased +- **Behaviour (0.8)**: request timeouts are the caller's, as OTP's + `gen_server:call`: a reply reads no clock, and only the caller's expiry + (its timer, the owner's deadline heap, a deadline check) times a request + out. A reply that loses that race is dropped (`Delivery::Expired`, the + caller sees `RequestTimeout`); one that lands after the deadline but + before the expiry runs is delivered. The `WRITING` slot state is gone and + no reply path reads the clock. `DeferredReplies.tla` models it (a reply + delivers while the slot is open; `LateSuccess` now overrides an expired + slot). +- Performance (0.8 request path, no API change): the reply slot keeps one + lock for value and waker with lazily created deferral state (no condvar: + a blocking `recv()` parks its thread); the tasks envelope lends its + request slot to the turn instead of cloning it; a turn that finishes in + its first poll skips the runtime-drop guard; `request` arms its first + timer from the timeout (no second clock read); a request without a + deadline holds no reference to the owner's registry (the owner's close + records the final reason in the slot); a caller that owns an answered + slot alone takes the value without the lock; and the slot no longer + holds its owner at all (two flags became word bits), which keeps it in + glibc's fast bins for typical reply sizes. Guarantees, Loom models and + the TLA+ models are unchanged (no linearization point moved). +- `InsertError::kind()` (a `#[non_exhaustive]` `InsertErrorKind`) and + `InsertError::into_parts()`: a refused insert's reason and token come + together, so a caller answers the token for any reason, including one + added later, instead of dropping it in a wildcard arm. +- Docs: an async job's `work` is a factory run at once in the current turn + (its panic is a handler panic; a job refused by the owner's exit has run + it); only its future is the job (`pipe` rustdoc, GUIDE). +- **Behaviour (0.8)**: `demonitor` flushes, as OTP's `demonitor(Ref, [flush])`: + once it returns, in both modes, no `Down` of that monitor is enqueued or + handled, including one already in the mailbox. Framework `Down`s travel in + a private envelope that checks the monitor table at dispatch, so ordinary + messages pay nothing and a `Down` user code sends is unaffected. New + failpoint site `monitor.down_before_dispatch` (#247). +- Fix: `PendingReplies::drain` yields live tokens in insertion order. It + walked a `HashMap` with a per-process `RandomState`, so deferred replies + were answered in a different order in each process, and a same-seed + simulator run did not reproduce its schedule (#261). `take` is now a + linear scan of the capped table. +- **Breaking (0.8), API pass before publishing** (each change is a row of the + [0.7 to 0.8 call-site table](docs/MIGRATION.md#07-to-08-api-call-site-table)): + - Jobs: one call per shape, options first (`pipe`, `pipe_blocking`, + `pipe_reply`, `pipe_reply_blocking`, `reserve_job(opts)`, + `reserve_blocking_job(opts)`); every `work` receives its `JobCtx` by + value. `pipe_with`, `pipe_with_ctx`, `pipe_keyed`, `pipe_blocking_with`, + `pipe_reply_with` and `pipe_reply_blocking_with` are gone. + `JobOptions` is `#[non_exhaustive]` with a builder (`new`, `key`, + `deadline(impl Into)`, `emit_credits(NonZeroU16)`, + `cancel_on_caller_gone`, which `debug_assert!`s outside `pipe_reply*`); + `deadline_after` is folded into `deadline`. `JobOutcome` is + `Result`. `PipeError` is the `#[non_exhaustive]` enum (no + `kind` wrapper); `PipeReplyError` is `#[non_exhaustive]` and its + `Failed` variant carries `error`. `Deadline` implements + `From` and `From`. `DEFAULT_EMIT_CREDITS` and + `default_max_native_jobs` are no longer public. + - Requests and deadlines: `request_raw` and `request_raw_until` become + `send_request(msg, impl Into)` (OTP's + `gen_server:send_request/2` with an explicit deadline; on `Receiver`, a `Deadline`). 0.7's + `request_raw` had no deadline: `Deadline::Never` keeps that. + `PendingReply::deadline`, `ReplyTo::deadline` and `ReplyReceiver::deadline` + return `Deadline`, so a deadline forwards as is; + `ReplyTo::with_deadline` is `shorten_deadline(impl Into)`. + Removed: `PendingReply::has_caller` (`defer()` returns `NoCaller`), + `PendingReplies::{purge, retained, live}` (now `len`/`is_empty`; purging + stays automatic), the public `DEFAULT_MAX_OUTSTANDING` (the default, + 1024, is documented at `Actor::MAX_OUTSTANDING`). + `PendingReply::detached` is hidden (`spawned_test::detached_reply()`). + `InsertError` holds `ReplyTo` and is `#[non_exhaustive]`. + `DEFAULT_REQUEST_TIMEOUT` is re-exported at the crate root. The + `Duration` parameters of `ActorSystem::shutdown`, `Supervisor::shutdown`, + `ShutdownReport::wait` and the test kit's waits are named `grace` or + `limit` (a deadline is an instant). `PendingReplies` and + `Context::deferred_outstanding` are documented as experimental. + - `#[non_exhaustive]` on the enums that will grow (`SupervisorError`, + `SystemError`, `HelperError`, `RegistryError`, `SupervisorStrategy`, + `ExecutionProfile`, `ChildState`, `Backend`, `spawned_test::Outcome`) + and on the read-only structs (`Down`, `Exit`, `ChildSnapshot`, + `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, + `HistogramSnapshot`). `Down` gains `actor`, the monitored `ActorId` + (OTP's `Object` in `{'DOWN', Ref, process, Object, Info}`). + - `SupervisorConfig`, `Backoff` and `metrics::Config` are + `#[non_exhaustive]` with builders (`new()` plus one setter per field; + `Backoff::seed(u64)`). + - Hidden and sealed surface: `Receiver` is sealed (it gained a + required method in 0.8; sealing lets the next one be added freely). The + `reply` module is `#[doc(hidden)]` (its types are at the crate root and + under `tasks`/`threads`; macros emit root paths). Test hooks + (`ActorRef::__in_turn`, `PendingReply::__discharge_stopped`, + `reply_pair`, `PendingReply::detached`) move into + `#[doc(hidden)] __private`, which spawned-test uses and which carries no + semver promise. `lifecycle_trace` is public only with the + `lifecycle-trace` feature; `spawned_rt::time`'s tick functions are + `#[doc(hidden)]`. + - Macros: `#[handler]` is removed (use `#[request_handler]` or + `#[send_handler]`), and a `#[send_handler]` method returning a value is + a compile error (it used to answer a `request` with it). + - Duplicates removed: `Context::from_ref` (use `ActorRef::context`) and + `ChildHandle::shutdown` (use `stop_with_reason(ExitReason::Shutdown)`). + - `Response` in tasks mode: `unwrap`, `expect`, `map` and the new + `into_result` on a response that was not awaited wait for the reply + when called off an async runtime and panic inside one (they always + panicked); `is_ok`/`is_err` panic on it instead of answering `false`. + - Docs: an "Experimental in 0.8" list in the GUIDE (job API, + `PendingReplies`, `exit_pending`, registry, helpers, placement + experiments, `spawned-rt` re-exports), the stop/exit/kill table on + `ChildHandle`, and `start_linked`'s unmanaged placement. The + `ActorError` classifiers (`target_gone`, `is_timeout`, `is_transient`, + `exit_reason`) are kept, defined per variant and tested for every one. +- Deferred replies and jobs, fixes from the 0.8 pre-release review: a job + deadline still fires while the result waits for scheduled emits, and the + job keeps its `MAX_OUTSTANDING` credit until its terminal message; a job + admitted while its owner exits is refused in one step with the exit + (`pipe_reply*` returns `OwnerStopping` with the token); in `tasks` mode a + blocking job's thread (outside a runtime) is created at reservation + (`SpawnFailed` instead of a silently lost job) and a managed actor's + blocking job keeps its scope until its body ends (a supervisor does not + replace the child, and shutdown is not complete, while it runs); + shortening a token's deadline no longer cancels a `cancel_on_caller_gone` + blocking job and wakes the waiting caller to re-arm its timer; an + `emit_at` discarded by a concurrent cancel returns `JobGone`; a refused + emit returns its credit once; a panicking payload destructor no longer + stops the process-wide deadline service; a tasks handler that panics + while building its future is a handler panic (`stopped()` runs). +- **Breaking (0.8):** `DeferError` and `Delivery` are `#[non_exhaustive]` + (like `JobError` and `PipeError`): match them with a wildcard arm. +- **Breaking (0.8):** the restart budget counts restart actions, as OTP's + intensity does (ROADMAP rank 5f): a OneForAll or RestForOne recovery round + takes one `max_restarts` grant when the coordinator selects it, before the + cohort is stopped, however many children it rebuilds (v0.7 charged every + rebuilt child, so a six-child OneForAll round needed six grants). A failed + replacement inside a round requests a new round, charged again (OTP's + `try_again_restart`); a superseded round keeps its charge; OneForOne and + retries before the tree is ready still take one grant per child retry. An + exhausted budget fails the supervisor when the round would be selected, + naming the round's origin child. `max_restarts` documentation, semantics + V5, the guide and the migration notes follow. +- The death point is one boundary, however the actor dies (run loop or + runtime drop): job admission closes, outgoing replies and `ctx.exit` + signals are fenced, the reason is sealed, and every deferred token and + queued request is answered `ActorStopped(final reason)` before any queued + payload or state destructor runs and before death is published. A + `ReplyTo::reply` once `NoProc` is observable returns + `Delivery::OwnerExited` (the caller gets the final reason); a signal a + cloned context sends before its actor's death reaches the target before + the death notice. Previously a blocking payload destructor, or a runtime + dropping the actor, could leave callers unanswered while it ran. +- **Breaking (0.8), in progress:** exit signals, startup and runtime-drop + semantics as in OTP (ROADMAP ranks 5e, 5f, 5g), implementing + [the design](docs/design/otp-exit-signals.md) (#240, stacked on rank 11). + Done so far: `ExitReason` is `#[non_exhaustive]` with `Killed` and `NoProc`; + `ChildHandle::kill()` (untrappable, ends `Killed`, supersedes any reason + until the death point), `ChildHandle::exit(reason)` and + `Context::exit(&target, reason)` with `erlang:exit_signal/2` rules + (`Normal` ignored by a non-trapping actor, other reasons end it without + `stopped()`, trapping actors get an `Exit`), `ActorId::EXTERNAL`; + administrative `stop`/`stop_with_reason`/`shutdown` unchanged; one death + point after `stopped()` where the reason becomes final and the link table + closes; a late untrapped signal replaces the requested reason unless a kill + was received (decision 8); a panic in `stopped()` replaces an earlier + abnormal reason (except after a kill); `stopped()` is skipped on a kill, an + untrapped signal or link notice and a runtime drop; runtime-dropped actors + end `Killed` in both modes; `link`/`monitor` to an actor past its death + point give `NoProc` (a non-trapping caller of `link` exits `NoProc`; + `Context::try_link` returns `Err(NoProc)`); `link` from an actor whose exit + began is inert; `unlink`/`demonitor` are fences; a `start_linked` parent's + exit ends a trapping child through `stopped()` (never `exit_received`); + signals arriving during `started()` are resolved when it returns; + supervisor and system shutdown send the parent's `Shutdown` exit signal + (a non-trapping child ends `Shutdown` without `stopped()`, a trapping one + finishes its backlog and runs `stopped()`), with the kill at + `shutdown_timeout` (supervisor, then a second interval before cleanup is + stuck) or at grace (system); destructor panics after + the death point are cleanup failures (`ChildHandle::cleanup_failure()`, + `mailbox_cleanup_failure()` deprecated) that restart a Transient child + without changing the reason; `spawned-test` `faults::exit` sends an exact + exit signal; metrics `actors_exited_killed_total`, + `actors_exited_noproc_total`. Supervisor startup (S1/S2): any failed first + start of a child (factory error or panic, panic or stop in `started()`, + `startup_timeout`, placement or group closure) fails `start()` whatever the + restart type, with no retry, as `SupervisorError::FailedToStartChild + { child, cause, rollback_failures }` (a stuck cleanup during the initial + start is `StartupCleanupStuck`); a ready child's exit during the initial + start is deferred and processed under the strategy once the tree commits + ready (OneForOne no longer restarts it at once, OneForAll/RestForOne no + longer fail the start); `start()` returns its error only after the + rollback (every child terminal, or `Failed` once its cleanup is stuck). + `Supervision.tla` (no restart before the tree is ready, deferral actions + `DeferExit`/`Undefer`, traced) and `Cohorts.tla` (round 0 once per child, + `FirstStartOnce`) follow, with the v0.7 behaviours as negative controls. + Caller errors (rank 11 x 5e, decision 4): `ActorError::ActorStopped` + carries the actor's final `ExitReason`, the same one `wait_exit`, `Down` + and links see (`Killed` after a kill, the signal's reason after an + untrapped exit signal, `Panic(..)` after a crash); new + `ActorError::NoProc` for a `send` or `request` after the death point (the + registry is sealed with the final reason inside the death point's + exit-table critical section); a send or request made while the actor exits + is admitted and answered at the death point. Exit errors wait for the + death point: a request whose handler panicked, a token dropped while the + owner exits and an undispatched queued request stay open ("exit pending") + and are answered with the final reason, or by the caller's deadline first. + `DeferredReplies.tla` models it (`pendA`, invariant `StoppedAtDeath`, + control `EarlyStopped`; one more reply trace control); metrics + `requests_noproc_total`. + Trace validation against `Lifecycle.tla`: with `lifecycle-trace`, the + runtime records exit signals, stops, links, monitors, sends, dispatch, + panics, `stopped()` run or skip, the death point, runtime drop, + publication and notice delivery (schema `lifecycle-v1`, + `TraceLifecycle.tla`); `tests/lifecycle_trace.rs` records 14 scenarios + (both modes), all accepted, and eleven edited traces (stopped() after a + kill, `Kill` for `Killed`, publication before D, real reason instead of + `NoProc`, notices after unlink/demonitor, ...) are rejected. + Client feedback (ethrex, ethlambda): `ActorError` classification helpers + `target_gone()`, `is_timeout()`, `is_transient()` and `exit_reason()`, so + clients need no exhaustive match on the `#[non_exhaustive]` enum; a + skipped `stopped()` logs `event = "stopped_skipped"` at `debug` (actor, + cause `kill`/`exit_signal`/`runtime_drop`, reason); + `Context::exit_pending()` (both modes) tells a trapping actor that its exit + began or a parent exit is queued behind the current message, so it can + refuse late work (advisory, read-only, no per-message cost). + Concurrency checks: the link table (`src/link/table.rs`) and the monitor + fence (`src/monitor/fence.rs`) are shared with a Loom model + (`tests/link_fence_loom.rs`, 9 tests explored exhaustively: one notice per + edge, none after `unlink`/`demonitor` or the receiver's death, a link or + monitor racing the death point gets the real reason or `NoProc`, never both + nor none); `tests/exit_signals_sim.rs` explores six orderings on the + simulator with PCT and bounded DFS (kill vs `stopped()`, late signal vs the + death point, link/monitor vs the death point, unlink/demonitor fences, + supervisor shutdown vs a trapping child's queue and `exit_pending`, a ready + child's exit vs the tree commit). GUIDE "Links, monitors and exit signals", + the `exit_signals` capability example, and semantics L3-L6, X1-X6, V1, V3, + V9 and the OTP differences table. + Remaining: the heavy TLC configurations and Apalache after the canaries. + +- **Breaking (0.8), in progress:** deferred replies and pipe-to-self + (ROADMAP rank 11), implementing + [the design](docs/design/deferred-replies.md). Every request now has a reply + slot: one packed atomic word (state, token, registry and running-job count) + whose transitions are checked with Loom (`reply_slot_loom`, 16 models), as + are the owner's turn-phase drop commit (`registry_turn_loom`) and the job + core (`job_core_loom`: admission rollback, completed-key replacement, emit + versus terminal notification and cancel, native completion versus cleanup + join, final destruction after a `Weak` upgrade), each with planted-bug + controls. The job model found two destructions under the job set's locks + (key-map clear at owner exit, weak-list compaction), now fixed. With + `lifecycle-trace`, the job core records `DeferredJobs.tla` actions, and + `tests/job_trace.rs` traces (seven scenarios) are checked by TLC + (`TraceJobs.tla`, schema `jobs-v1`), with six edited traces as negative + controls. Reply slots record `DeferredReplies.tla` actions (send, + dispatch, the terminal transition inside the slot CAS, exit, death); + `tests/reply_trace.rs` traces (nine scenarios) are checked against + `TraceReplies.tla`, with four negative controls. With `metrics`, the + `spawned_deferred_*` family (tokens, pending, replied/expired/dropped/ + stopped/abandoned, overloaded, sampled defer-to-reply latency) and the + `spawned_jobs_*` family (started, running, completed/panicked/deadline + exceeded, cancelled while running, fenced, rejected at a bound, emits, + unfinished at stop); `tests/metrics_deferred.rs` checks each is counted + once. Simulator races (`tests/deferred_sim.rs`, PCT and bounded DFS): + reply vs expiry vs owner death vs caller drop, keyed replacement, emit vs + terminal and cancel. `Handler::handle` takes a `PendingReply<'_, R>` + and returns `()`; `#[actor]` generates it, and `#[deferred_handler]` methods + receive the obligation and may `defer()` it into a storable `ReplyTo` + (bounded by `Actor::MAX_OUTSTANDING`, `Overloaded` at the bound). Dropped + obligations answer `ReplyDropped` (in a panicking turn or while exiting + they are answered `ActorStopped` at death); at death every open token + answers `ActorStopped`. Request + deadlines are fixed at send (`Deadline`, `send_request(msg, deadline)`, + formerly `request_raw`); a per-actor deadline heap and service expire stored + tokens without a polling caller. `PendingReplies` is a bounded token + table. Pipe-to-self jobs: `ctx.pipe`, `pipe_blocking`, `pipe_reply`, + `pipe_reply_blocking` (threads mode: the blocking forms on owned native + threads), with `JobOutcome`/`JobError::{Panicked, DeadlineExceeded}`, keyed + replacement and cancellation fenced at dispatch, `JobCtx::emit`/ + `emit_blocking` with emit credits and the terminal message last, + `MAX_NATIVE_JOBS`, owner-exit cancellation and a bounded join after death + (`unfinished_at_stop`). `ActorError` is `#[non_exhaustive]` with + `ReplyDropped` and `Overloaded`; `metrics::Snapshot` gains + `requests_reply_dropped_total` and `requests_overloaded_total`. Example: + `deferred_replies`. See [migration](docs/MIGRATION.md#unreleased). +- Rank 11 API refinements from porting ethlambda and ethrex to 0.8: + `#[actor(max_outstanding = N, max_native_jobs = N)]` sets the bounds, and + `ChildSpec::max_outstanding`/`max_native_jobs` override them per + supervised child (the design's per-spec override). `tasks`-mode blocking + jobs (`pipe_blocking*`) run on Tokio's blocking pool instead of a fresh OS + thread each, still under `MAX_NATIVE_JOBS`, with panics as + `JobError::Panicked` and cancellation and fencing unchanged (threads mode + keeps owned threads; a reservation pre-spawns the job's thread). + Reserve-then-spawn: `ctx.reserve_job(opts)` / `ctx.reserve_blocking_job(opts)` + return a `JobPermit` / `BlockingJobPermit` whose `pipe*` cannot fail, so a + refused caller keeps what it would have moved into `work` and `done`. + `JobCtx::emit_at` / `emit_at_blocking` deliver a fenced, credit-accounted + message at an instant; the job's terminal message waits for its scheduled + emits (new Loom model `scheduled_emit_vs_finish_and_cancel`). + `spawned_concurrency::Instant` + (also `tasks::`/`threads::Instant`) re-exports the deadline clock. + `#[send_handler]` on a manual `impl Handler` (for `Result = ()`) drops + the reply parameter and answers `()` when `handle` returns. + `spawned_test::tasks::in_actor` / `threads::in_actor` run a closure in a + live actor's turn. GUIDE: hard job deadlines and the soft-deadline + pattern, replies carrying a reason through `R = Result<..>`, the + `send_request` rows. - Design (0.9, approved for implementation after round 3): a unified start API (`docs/design/unified-start.md`, rank 5h). OTP's `start`, `start_link` and `start_monitor` wait for `started()`, and `spawn` does not. All four take one diff --git a/Cargo.lock b/Cargo.lock index e960fe242..4395d8d0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1000,6 +1000,7 @@ checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ "cfg-if", "generator", + "pin-utils", "scoped-tls", "tracing", "tracing-subscriber", @@ -1291,6 +1292,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13bee6c73da26345c729282832b60b0363cf3dd9f4bfd81d8551b7a1c889a113" + [[package]] name = "ping_pong" version = "0.1.0" diff --git a/README.md b/README.md index f7ca25978..3c727da53 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ The return type on each protocol method determines the message kind: - `Result<(), ActorError>` โ€” fire-and-forget send (both modes), returns the send result - No return / `-> ()` โ€” fire-and-forget send (both modes), discards the send result -**Actors** implement message handlers with `#[actor]`. Each handler method is annotated with `#[request_handler]`, `#[send_handler]`, or `#[handler]` and receives a single message struct plus a `Context`. The macro generates the `Actor` trait impl and one `Handler` impl per method. +**Actors** implement message handlers with `#[actor]`. Each handler method is annotated with `#[request_handler]` (answers with its return value), `#[send_handler]` (returns `()`) or `#[deferred_handler]` (answers through its `PendingReply`, now or later) and receives a single message struct plus a `Context`. The macro generates the `Actor` trait impl and one `Handler` impl per method. **Using an actor** is just calling trait methods on its `ActorRef`. Because `#[protocol]` generates `impl NameServerProtocol for ActorRef`, you write `ns.add(...)` and `ns.find(...)` โ€” the macro handles message construction and mailbox dispatch behind the scenes. diff --git a/ROADMAP.md b/ROADMAP.md index 2931b4e06..75b2b2dd1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -720,7 +720,7 @@ Small, independent of the scheduler and lifecycle design, and visible to users. |---|---|---| | 5d | One semantics contract | A single `docs/design/semantics.md` stating every guarantee Spawned gives (message ordering and delivery, request outcomes, exit propagation through links and monitors, panic handling, restart and budget rules, shutdown order, timers, what `panic=abort` disables), each linked to the TLA+ invariant, test or example that checks it, and listing what is explicitly not guaranteed. Today these are spread across the guide, design contracts, TESTING.md and the models. New designs are reviewed against it. Erlang's semantics fit on a few pages; ours should too. | | 5e | Exit signals as in OTP (v0.8) | Align every exit-signal rule with OTP, as one design with a TLA+ model (the planned base-lifecycle model: links, monitors, `trap_exit`) and negative controls. (1) A direct `kill` stays untrappable, but the exit it causes propagates to linked actors as a distinct `Killed` reason (OTP `killed`), which they can trap. (2) `exit(Pid, normal)` semantics: a `Normal` exit signal from another actor is ignored by an actor that is not trapping exits; other reasons, including `Shutdown`, are delivered as trappable `Exit` messages to a trapping actor, and only `Kill` cannot be trapped. Today every direct stop ignores `trap_exit`. (3) A supervisor's `Shutdown` to a trapping child runs its normal termination path, as gen_server calls `terminate` on an exit from its parent. (4) Linking to or monitoring an actor that has already exited reports `NoProc` (OTP `noproc`) instead of its earlier exit reason. Breaking for exhaustive matches on `ExitReason`: pair with `#[non_exhaustive]` (as rank 11 does for `ActorError`), and update the semantics contract, migration notes and trace mapping. | -| 5f | Restart budget counts recovery rounds (OTP alignment, v0.8) | Today every rebuilt child counts against the budget, including the siblings OneForAll and RestForOne rebuild, so those strategies exhaust a budget N times faster than OTP users expect. Count one restart per recovery round, as OTP does. Also align initial +| 5f | Restart budget counts recovery rounds (OTP alignment, v0.8) | **Done in the 0.8 stack (#240):** a OneForAll/RestForOne recovery round takes one grant when the coordinator selects it (OTP's `add_restart` once per restart action); its rebuilds take none; a failed replacement requests a new, charged round. Before, every rebuilt child counted, so those strategies exhausted a budget N times faster than OTP users expect. Remaining: `Cohorts.tla` still abstracts budgets (`Supervision.tla`'s `BudgetBounded` covers OneForOne, where a child retry is the restart action); modelling the per-round charge with a per-child negative control is follow-up work. Also align initial startup: OTP aborts `start_link` on any child start error, while Spawned retries a Permanent or Transient child's failed factory, panic or startup timeout during the first start (semantics ยง11; found reviewing #233). Behaviour change: extend the `Supervision`/`Cohorts` TLA+ models and their Apalache inductive checks (`BudgetBounded`) to the new rule, with a negative control for counting per child, and update the Kani restart-window proofs and the guide. | diff --git a/concurrency/Cargo.toml b/concurrency/Cargo.toml index 6606d2929..0872cfc10 100644 --- a/concurrency/Cargo.toml +++ b/concurrency/Cargo.toml @@ -83,7 +83,7 @@ required-features = ["backend-experiments"] pprof = { version = "0.15", optional = true, features = ["flamegraph"] } [target.'cfg(spawned_scheduler_loom)'.dev-dependencies] -loom = "0.7" +loom = { version = "0.7", features = ["futures"] } [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ['cfg(spawned_scheduler_loom)', 'cfg(spawned_sim)', 'cfg(kani)', 'cfg(kani_slow)'] } diff --git a/concurrency/benches/baseline.rs b/concurrency/benches/baseline.rs index 0f8ca9c0c..2ace42c13 100644 --- a/concurrency/benches/baseline.rs +++ b/concurrency/benches/baseline.rs @@ -1,9 +1,10 @@ //! Process-isolated baseline driver; methodology in benchmarks/README.md. #[path = "support/args.rs"] mod cli; -use spawned_concurrency::{message::Message, tasks, threads, ChildHandle}; +use spawned_concurrency::{message::Message, tasks, threads, ChildHandle, Deadline, ReplyReceiver}; use std::{ alloc::{GlobalAlloc, Layout, System}, + future::IntoFuture, hint::black_box, sync::{ atomic::{AtomicBool, AtomicU64, Ordering}, @@ -98,14 +99,24 @@ fn process(work: Work) -> Ack { struct Sink; impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> Ack { - process(work) + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, Ack>, + _: &tasks::Context, + ) { + reply.reply(process(work)); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> Ack { - process(work) + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, Ack>, + _: &threads::Context, + ) { + reply.reply(process(work)); } } #[derive(Clone)] @@ -115,19 +126,30 @@ enum Target { Tokio(tokio::sync::mpsc::UnboundedSender<(Work, tokio::sync::oneshot::Sender)>), } enum Reply { - Tasks(tokio::sync::oneshot::Receiver), - Threads(mpsc::Receiver), + Tasks(ReplyReceiver), + Tokio(tokio::sync::oneshot::Receiver), + Threads(ReplyReceiver), } impl Target { + /// A raw request with no deadline: the 0.8 form of 0.7's `request_raw` + /// (no timeout), so the comparison with 0.7 measures the same operation. fn request(&self, work: Work) -> Reply { match self { - Self::Tasks(actor) => Reply::Tasks(actor.request_raw(work).expect("actor stopped")), - Self::Threads(actor) => Reply::Threads(actor.request_raw(work).expect("actor stopped")), + Self::Tasks(actor) => Reply::Tasks( + actor + .send_request(work, Deadline::Never) + .expect("actor stopped"), + ), + Self::Threads(actor) => Reply::Threads( + actor + .send_request(work, Deadline::Never) + .expect("actor stopped"), + ), Self::Tokio(tx) => { let (reply, rx) = tokio::sync::oneshot::channel(); tx.send((work, reply)) .unwrap_or_else(|_| panic!("baseline stopped")); - Reply::Tasks(rx) + Reply::Tokio(rx) } } } @@ -135,15 +157,16 @@ impl Target { impl Reply { fn wait(self, rt: &tokio::runtime::Handle) -> Ack { match self { - Self::Tasks(rx) => rt.block_on(async { + Self::Tasks(rx) => rt + .block_on(rx.into_future()) + .expect("reply timeout or dropped"), + Self::Tokio(rx) => rt.block_on(async { tokio::time::timeout(Duration::from_secs(10), rx) .await .expect("reply timeout") .expect("reply dropped") }), - Self::Threads(rx) => rx - .recv_timeout(Duration::from_secs(10)) - .expect("reply timeout/disconnected"), + Self::Threads(rx) => rx.recv().expect("reply timeout/disconnected"), } } } diff --git a/concurrency/benches/designs.rs b/concurrency/benches/designs.rs index 1ee24726f..178147adc 100644 --- a/concurrency/benches/designs.rs +++ b/concurrency/benches/designs.rs @@ -123,41 +123,54 @@ impl Sink { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> Ack { - let start = if work.timing { - Instant::now() - } else { - work.enqueued - }; - match self.work.as_str() { - "timer" => tokio::time::sleep(self.wait).await, - "blocking" => std::thread::sleep(self.wait), - "tcp" => { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - if self.socket.is_none() { - let socket = tokio::net::TcpStream::connect(self.address.unwrap()) - .await - .unwrap(); - socket.set_nodelay(true).unwrap(); - self.socket = Some(socket); + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, Ack>, + _: &tasks::Context, + ) { + let __reply_value: Ack = { + let start = if work.timing { + Instant::now() + } else { + work.enqueued + }; + match self.work.as_str() { + "timer" => tokio::time::sleep(self.wait).await, + "blocking" => std::thread::sleep(self.wait), + "tcp" => { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + if self.socket.is_none() { + let socket = tokio::net::TcpStream::connect(self.address.unwrap()) + .await + .unwrap(); + socket.set_nodelay(true).unwrap(); + self.socket = Some(socket); + } + let socket = self.socket.as_mut().unwrap(); + let mut frame = [0u8; 64]; + frame[..8].copy_from_slice(&work.id.to_le_bytes()); + let expected = frame; + socket.write_all(&frame).await.unwrap(); + socket.read_exact(&mut frame).await.unwrap(); + assert_eq!(frame, expected); } - let socket = self.socket.as_mut().unwrap(); - let mut frame = [0u8; 64]; - frame[..8].copy_from_slice(&work.id.to_le_bytes()); - let expected = frame; - socket.write_all(&frame).await.unwrap(); - socket.read_exact(&mut frame).await.unwrap(); - assert_eq!(frame, expected); + _ => {} } - _ => {} - } - process_at(work, start) + process_at(work, start) + }; + reply.reply(__reply_value); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> Ack { - process(work) + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, Ack>, + _: &threads::Context, + ) { + reply.reply(process(work)); } } enum ReplySender { @@ -188,13 +201,29 @@ impl Message for Probe { type Result = (); } impl tasks::Handler for Sink { - async fn handle(&mut self, probe: Probe, _: &tasks::Context) { - probe.reply.send(process(probe.work)); + async fn handle( + &mut self, + probe: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &tasks::Context, + ) { + { + probe.reply.send(process(probe.work)); + } + reply.reply(()); } } impl threads::Handler for Sink { - fn handle(&mut self, probe: Probe, _: &threads::Context) { - probe.reply.send(process(probe.work)); + fn handle( + &mut self, + probe: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &threads::Context, + ) { + { + probe.reply.send(process(probe.work)); + } + reply.reply(()); } } enum TokioMessage { @@ -264,6 +293,9 @@ enum Target { enum Reply { Tasks(tokio::sync::oneshot::Receiver), Threads(mpsc::Receiver), + /// A framework request's receiver (its deadline is fixed at send). + FrameworkTasks(spawned_concurrency::ReplyReceiver), + FrameworkThreads(spawned_concurrency::ReplyReceiver), } impl Target { fn request(&self, work: Work, kind: &str) -> Reply { @@ -282,8 +314,16 @@ impl Target { return rx; } match self { - Self::Tasks(actor) => Reply::Tasks(actor.request_raw(work).expect("actor stopped")), - Self::Threads(actor) => Reply::Threads(actor.request_raw(work).expect("actor stopped")), + Self::Tasks(actor) => Reply::FrameworkTasks( + actor + .send_request(work, Duration::from_secs(10)) + .expect("actor stopped"), + ), + Self::Threads(actor) => Reply::FrameworkThreads( + actor + .send_request(work, Duration::from_secs(10)) + .expect("actor stopped"), + ), Self::Tokio(tx) => { let (reply, rx) = tokio::sync::oneshot::channel(); tx.send(TokioMessage::Request(work, reply)) @@ -300,6 +340,11 @@ impl Reply { return match self { Self::Tasks(rx) => futures::executor::block_on(rx).expect("reply dropped"), Self::Threads(rx) => rx.recv().expect("reply dropped"), + Self::FrameworkTasks(rx) => { + futures::executor::block_on(std::future::IntoFuture::into_future(rx)) + .expect("reply failed") + } + Self::FrameworkThreads(rx) => rx.recv().expect("reply failed"), }; } match self { @@ -312,6 +357,8 @@ impl Reply { Self::Threads(rx) => rx .recv_timeout(Duration::from_secs(10)) .expect("reply timeout/disconnected"), + Self::FrameworkTasks(rx) => rt.block_on(async { rx.await.expect("reply failed") }), + Self::FrameworkThreads(rx) => rx.recv().expect("reply failed"), } } } diff --git a/concurrency/benches/lifecycle_soak.rs b/concurrency/benches/lifecycle_soak.rs index d31d88001..716a24323 100644 --- a/concurrency/benches/lifecycle_soak.rs +++ b/concurrency/benches/lifecycle_soak.rs @@ -61,23 +61,31 @@ impl Message for Step { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Step, _: &Context) -> u64 { - assert_eq!(self.state, msg.expected); - assert_eq!(self.bytes[0], 13); - if msg.helper { - assert_eq!( - tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; + async fn handle( + &mut self, + msg: Step, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + assert_eq!(self.state, msg.expected); + assert_eq!(self.bytes[0], 13); + if msg.helper { + assert_eq!( + tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }) + .await + .unwrap(), 42 - }) - .await - .unwrap(), - 42 - ); - } - self.state += 1; - self.counts.requests.fetch_add(1, Ordering::Relaxed); - self.state + ); + } + self.state += 1; + self.counts.requests.fetch_add(1, Ordering::Relaxed); + self.state + }; + reply.reply(__reply_value); } } struct Crash; @@ -85,7 +93,12 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("soak injected handler failure"); } } @@ -99,14 +112,25 @@ impl Message for Watch { type Result = (); } impl Handler for Watcher { - async fn handle(&mut self, msg: Watch, ctx: &Context) { + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { for (child, panic) in msg.0 { assert!(self.pending.insert(ctx.monitor(&child), panic).is_none()); } + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Down, _: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let panic = self .pending .remove(&msg.monitor_ref) @@ -116,6 +140,7 @@ impl Handler for Watcher { assert_eq!(msg.reason, ExitReason::Normal); } self.counts.down.fetch_add(1, Ordering::Release); + reply.reply(()); } } struct Remaining; @@ -123,8 +148,13 @@ impl Message for Remaining { type Result = usize; } impl Handler for Watcher { - async fn handle(&mut self, _: Remaining, _: &Context) -> usize { - self.pending.len() + async fn handle( + &mut self, + _: Remaining, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.pending.len()); } } fn pin(cpus: &str) { diff --git a/concurrency/benches/metrics_overhead.rs b/concurrency/benches/metrics_overhead.rs index 6eba29d98..7b67de52a 100644 --- a/concurrency/benches/metrics_overhead.rs +++ b/concurrency/benches/metrics_overhead.rs @@ -18,14 +18,24 @@ fn compute(work: Work) -> u64 { } impl tasks::Actor for Sink {} impl tasks::Handler for Sink { - async fn handle(&mut self, work: Work, _: &tasks::Context) -> u64 { - compute(work) + async fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &tasks::Context, + ) { + reply.reply(compute(work)); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, work: Work, _: &threads::Context) -> u64 { - compute(work) + fn handle( + &mut self, + work: Work, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &threads::Context, + ) { + reply.reply(compute(work)); } } fn main() { @@ -44,9 +54,9 @@ fn main() { assert!(actors > 0 && actors <= 1024 && producers > 0 && producers <= 1024); assert!(duration.as_secs() > 0 && duration.as_secs() <= 60 && work <= 1_000_000); #[cfg(feature = "metrics")] - spawned_concurrency::metrics::configure(spawned_concurrency::metrics::Config { - timing_sample_every: _sample_every, - }) + spawned_concurrency::metrics::configure( + spawned_concurrency::metrics::Config::default().timing_sample_every(_sample_every), + ) .unwrap(); let (requests, elapsed) = if mode == "async" { use tasks::ActorStart; diff --git a/concurrency/benches/pinned_pipeline.rs b/concurrency/benches/pinned_pipeline.rs index c024b39ee..2e1378d60 100644 --- a/concurrency/benches/pinned_pipeline.rs +++ b/concurrency/benches/pinned_pipeline.rs @@ -49,8 +49,14 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, m: Wire, _: &Context) { + async fn handle( + &mut self, + m: Wire, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.next = m.0; + reply.reply(()); } } struct Work { @@ -63,19 +69,27 @@ mod linux { type Result = (); } impl Handler for Stage { - async fn handle(&mut self, mut m: Work, _: &Context) { - for _ in 0..self.iterations { - m.value = black_box(m.value.wrapping_mul(MUL).wrapping_add(1)); - } - m.hops += 1; - self.handled += 1; - if let Some(next) = &self.next { - next.send(m).unwrap(); - } else { - m.reply - .send((m.id, m.value, m.hops, Instant::now())) - .unwrap(); + async fn handle( + &mut self, + mut m: Work, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + for _ in 0..self.iterations { + m.value = black_box(m.value.wrapping_mul(MUL).wrapping_add(1)); + } + m.hops += 1; + self.handled += 1; + if let Some(next) = &self.next { + next.send(m).unwrap(); + } else { + m.reply + .send((m.id, m.value, m.hops, Instant::now())) + .unwrap(); + } } + reply.reply(()); } } struct Audit; @@ -83,19 +97,27 @@ mod linux { type Result = (String, String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, _: &Context) -> (String, String, u64) { - let status = fs::read_to_string("/proc/thread-self/status").unwrap(); - let affinity = status - .lines() - .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) - .unwrap() - .trim() - .to_owned(); - ( - std::thread::current().name().unwrap_or("main").to_owned(), - affinity, - self.handled, - ) + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::PendingReply<'_, (String, String, u64)>, + _: &Context, + ) { + let __reply_value: (String, String, u64) = { + let status = fs::read_to_string("/proc/thread-self/status").unwrap(); + let affinity = status + .lines() + .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .to_owned(); + ( + std::thread::current().name().unwrap_or("main").to_owned(), + affinity, + self.handled, + ) + }; + reply.reply(__reply_value); } } fn pin(mask: &str) { diff --git a/concurrency/benches/runtime_isolation.rs b/concurrency/benches/runtime_isolation.rs index df8981c66..641b7c5ee 100644 --- a/concurrency/benches/runtime_isolation.rs +++ b/concurrency/benches/runtime_isolation.rs @@ -46,16 +46,29 @@ mod linux { type Result = (u64, u64, Instant); } impl Handler for Stage { - async fn handle(&mut self, m: Work, _: &Context) -> (u64, u64, Instant) { - self.handled += 1; - if let Some(next) = &self.next { - return next.request_raw(m).unwrap().await.unwrap(); - } - let mut value = m.0; - for _ in 0..self.iterations { - value = black_box(value.wrapping_mul(MUL).wrapping_add(1)); + async fn handle( + &mut self, + m: Work, + reply: spawned_concurrency::PendingReply<'_, (u64, u64, Instant)>, + _: &Context, + ) { + let __reply_value: (u64, u64, Instant) = async { + self.handled += 1; + if let Some(next) = &self.next { + return next + .send_request(m, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); + } + let mut value = m.0; + for _ in 0..self.iterations { + value = black_box(value.wrapping_mul(MUL).wrapping_add(1)); + } + (m.0, value, Instant::now()) } - (m.0, value, Instant::now()) + .await; + reply.reply(__reply_value); } } struct Audit; @@ -63,17 +76,25 @@ mod linux { type Result = (String, u64); } impl Handler for Stage { - async fn handle(&mut self, _: Audit, _: &Context) -> (String, u64) { - let status = fs::read_to_string("/proc/thread-self/status").unwrap(); - ( - status - .lines() - .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) - .unwrap() - .trim() - .into(), - self.handled, - ) + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::PendingReply<'_, (String, u64)>, + _: &Context, + ) { + let __reply_value: (String, u64) = { + let status = fs::read_to_string("/proc/thread-self/status").unwrap(); + ( + status + .lines() + .find_map(|s| s.strip_prefix("Cpus_allowed_list:")) + .unwrap() + .trim() + .into(), + self.handled, + ) + }; + reply.reply(__reply_value); } } fn pin(mask: &str) { @@ -199,7 +220,11 @@ mod linux { barrier.wait().await; let mut n = 0u64; while Instant::now() < *finish.get().unwrap() { - let reply = actor.request_raw(Work(n)).unwrap().await.unwrap(); + let reply = actor + .send_request(Work(n), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); check(reply, n, ab, bb); n += 1; } @@ -219,7 +244,11 @@ mod linux { let mut n = 0; while Instant::now() < *cfinish.get().unwrap() { let start = Instant::now(); - let reply = endpoint.request_raw(Work(n)).unwrap().await.unwrap(); + let reply = endpoint + .send_request(Work(n), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + .await + .unwrap(); let end = Instant::now(); check(reply, n, ac, bc); latency.push(end.duration_since(start).as_nanos() as u64); diff --git a/concurrency/benches/soak/main.rs b/concurrency/benches/soak/main.rs index ee088e5e7..0eb5cb1b8 100644 --- a/concurrency/benches/soak/main.rs +++ b/concurrency/benches/soak/main.rs @@ -361,9 +361,7 @@ fn main() { }; LEAKS.set(args.leaks).unwrap(); // Counters and gauges only: no sampled clocks. - let _ = metrics::configure(metrics::Config { - timing_sample_every: 0, - }); + let _ = metrics::configure(metrics::Config::new().timing_sample_every(0)); // Injected crashes are expected millions of times; keep other panics loud. let default_hook = std::panic::take_hook(); std::panic::set_hook(Box::new(move |info| { diff --git a/concurrency/benches/soak/native.rs b/concurrency/benches/soak/native.rs index f102b86c5..7975b460c 100644 --- a/concurrency/benches/soak/native.rs +++ b/concurrency/benches/soak/native.rs @@ -50,33 +50,69 @@ impl Message for Bump { type Result = (); } impl Handler for NWorker { - fn handle(&mut self, _: Inc, _: &Context) -> u64 { - self.value += 1; - self.value + fn handle( + &mut self, + _: Inc, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { + self.value += 1; + self.value + }; + reply.reply(value); } } impl Handler for NWorker { - fn handle(&mut self, _: Get, _: &Context) -> u64 { - self.value + fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { self.value }; + reply.reply(value); } } impl Handler for NWorker { - fn handle(&mut self, _: Bump, _: &Context) { + fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.value += 1; + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("soak: injected crash"); } } impl Handler for NWorker { - fn handle(&mut self, msg: Slow, _: &Context) { + fn handle( + &mut self, + msg: Slow, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { std::thread::sleep(msg.0); + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + fn handle( + &mut self, + msg: ArmTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { let ArmTimers(once, every, period) = msg; for i in 0..once { let delay = period * (i as u32 + 1); @@ -85,13 +121,20 @@ impl Handler for NWorker { for _ in 0..every { self.timers.push(send_interval(period, ctx.clone(), Bump)); } + reply.reply(()); } } impl Handler for NWorker { - fn handle(&mut self, _: CancelTimers, _: &Context) { + fn handle( + &mut self, + _: CancelTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { for timer in self.timers.drain(..) { timer.cancellation_token.cancel(); } + reply.reply(()); } } @@ -114,33 +157,69 @@ message!(LinkTo(ChildHandle) => ()); message!(Trap(bool) => ()); message!(Counts => (u64, u64)); impl Handler for NWatcher { - fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { - ctx.monitor(&msg.0) + fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + let value = { ctx.monitor(&msg.0) }; + reply.reply(value); } } impl Handler for NWatcher { - fn handle(&mut self, msg: Unwatch, ctx: &Context) { + fn handle( + &mut self, + msg: Unwatch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, msg: LinkTo, ctx: &Context) { + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, msg: Trap, ctx: &Context) { + fn handle( + &mut self, + msg: Trap, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(msg.0); + reply.reply(()); } } impl Handler for NWatcher { - fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { - (self.downs, self.exits) + fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::PendingReply<'_, (u64, u64)>, + _: &Context, + ) { + let value = { (self.downs, self.exits) }; + reply.reply(value); } } impl Handler for NWatcher { - fn handle(&mut self, _: Down, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.downs += 1; + reply.reply(()); } } @@ -336,8 +415,8 @@ fn registry_requests(rng: &mut Rng, stats: &Stats) -> R { ensure!(actor.request(Inc).map_err(e("inc"))? == 1, "inc"); finish(&actor, rng)?; ensure!( - matches!(actor.request(Inc), Err(ActorError::ActorStopped)), - "request to a stopped actor" + matches!(actor.request(Inc), Err(ActorError::NoProc)), + "request to a stopped actor (NoProc past its death point)" ); Ok(()) } diff --git a/concurrency/benches/soak/workload.rs b/concurrency/benches/soak/workload.rs index 63be5c5d6..20bcd948f 100644 --- a/concurrency/benches/soak/workload.rs +++ b/concurrency/benches/soak/workload.rs @@ -87,33 +87,69 @@ impl Message for Bump { } impl Handler for Worker { - async fn handle(&mut self, _: Inc, _: &Context) -> u64 { - self.value += 1; - self.value + async fn handle( + &mut self, + _: Inc, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { + self.value += 1; + self.value + }; + reply.reply(value); } } impl Handler for Worker { - async fn handle(&mut self, _: Get, _: &Context) -> u64 { - self.value + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let value = { self.value }; + reply.reply(value); } } impl Handler for Worker { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.value += 1; + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("soak: injected crash"); } } impl Handler for Worker { - async fn handle(&mut self, msg: Slow, _: &Context) { + async fn handle( + &mut self, + msg: Slow, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { rt::sleep(msg.0).await; + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: ArmTimers, ctx: &Context) { + async fn handle( + &mut self, + msg: ArmTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { let ArmTimers(once, every, period) = msg; for i in 0..once { let delay = period * (i as u32 + 1); @@ -122,18 +158,31 @@ impl Handler for Worker { for _ in 0..every { self.timers.push(send_interval(period, ctx.clone(), Bump)); } + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: CancelTimers, _: &Context) { + async fn handle( + &mut self, + _: CancelTimers, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { for timer in self.timers.drain(..) { timer.cancellation_token.cancel(); } + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: RegisterManaged, ctx: &Context) -> Result<(), String> { - ctx.register_name(&msg.0).map_err(|e| e.to_string()) + async fn handle( + &mut self, + msg: RegisterManaged, + reply: spawned_concurrency::PendingReply<'_, Result<(), String>>, + ctx: &Context, + ) { + let value = { ctx.register_name(&msg.0).map_err(|e| e.to_string()) }; + reply.reply(value); } } @@ -168,43 +217,91 @@ message!(Trap(bool) => ()); message!(Counts => (u64, u64)); message!(SpawnLinked => ActorRef); impl Handler for Watcher { - async fn handle(&mut self, msg: Watch, ctx: &Context) -> MonitorRef { - ctx.monitor(&msg.0) + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + let value = { ctx.monitor(&msg.0) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Unwatch, ctx: &Context) { + async fn handle( + &mut self, + msg: Unwatch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Unlink, ctx: &Context) { + async fn handle( + &mut self, + msg: Unlink, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.unlink(&msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: Trap, ctx: &Context) { + async fn handle( + &mut self, + msg: Trap, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, _: Counts, _: &Context) -> (u64, u64) { - (self.downs, self.exits) + async fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::PendingReply<'_, (u64, u64)>, + _: &Context, + ) { + let value = { (self.downs, self.exits) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, _: SpawnLinked, ctx: &Context) -> ActorRef { - Worker::new().start_linked(ctx) + async fn handle( + &mut self, + _: SpawnLinked, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, + ctx: &Context, + ) { + let value = { Worker::new().start_linked(ctx) }; + reply.reply(value); } } impl Handler for Watcher { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.downs += 1; + reply.reply(()); } } @@ -417,17 +514,15 @@ async fn group(rng: &mut Rng, env: &Env) -> R { } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 60_000, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::new() + .strategy(strategy) + .max_restarts(60_000) + .backoff( + Backoff::new() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ) } fn child(name: String, group: &ExecutionGroup) -> ChildSpec { @@ -689,8 +784,8 @@ async fn requests(rng: &mut Rng) -> R { ensure!(got == 1, "inc after timeouts returned {got}"); finish(&actor, rng).await?; ensure!( - matches!(actor.request(Inc).await, Err(ActorError::ActorStopped)), - "request to a stopped actor" + matches!(actor.request(Inc).await, Err(ActorError::NoProc)), + "request to a stopped actor (NoProc past its death point)" ); ensure!(actor.send(Bump).is_err(), "send to a stopped actor"); Ok(()) diff --git a/concurrency/benches/workload_research.rs b/concurrency/benches/workload_research.rs index 8086108f9..de88d1c95 100644 --- a/concurrency/benches/workload_research.rs +++ b/concurrency/benches/workload_research.rs @@ -1,4 +1,5 @@ //! Full actors, explicit DAG placement and independently scheduled offered load. +use std::future::IntoFuture as _; #[path = "support/args.rs"] mod cli; #[path = "support/profiling.rs"] @@ -230,7 +231,12 @@ struct Node { } impl Actor for Node {} impl Handler for Node { - async fn handle(&mut self, msg: Wire, _: &Context) { + async fn handle( + &mut self, + msg: Wire, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { if std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").as_deref() == Ok("sharded") { assert_eq!( std::thread::current().name().unwrap(), @@ -238,6 +244,7 @@ impl Handler for Node { ); } self.children = msg.children; + reply.reply(()); } } #[inline(never)] @@ -248,41 +255,55 @@ fn cpu_work(mut value: u64, iterations: u32) -> u64 { value } impl Handler for Node { - async fn handle(&mut self, msg: Route, _: &Context) -> Ack { - let started = Instant::now(); - if msg.measured { - self.meter.started.fetch_add(1, Ordering::Relaxed); - } - let value = cpu_work(msg.id ^ (self.id as u64), msg.iterations); - // Start all fan-out requests before awaiting any; chains have one child. - let replies: Vec<_> = self - .children - .iter() - .map(|c| c.request_raw(msg.clone()).unwrap()) - .collect(); - let mut nodes = 1; - let mut tag = self.id as u64 + 1; - let mut checksum = value; - for reply in futures::future::join_all(replies).await { - let ack = reply.unwrap(); - assert_eq!(ack.id, msg.id); - nodes += ack.nodes; - tag += ack.tag; - checksum ^= ack.checksum; - } - let completed = Instant::now(); - if msg.measured { - self.meter.completed.fetch_add(1, Ordering::Relaxed); - } - Ack { - id: msg.id, - nodes, - tag, - checksum, - completed, - queue_ns: started.duration_since(msg.enqueued).as_nanos() as u64, - service_ns: completed.duration_since(started).as_nanos() as u64, - } + async fn handle( + &mut self, + msg: Route, + reply: spawned_concurrency::PendingReply<'_, Ack>, + _: &Context, + ) { + let __reply_value: Ack = { + let started = Instant::now(); + if msg.measured { + self.meter.started.fetch_add(1, Ordering::Relaxed); + } + let value = cpu_work(msg.id ^ (self.id as u64), msg.iterations); + // Start all fan-out requests before awaiting any; chains have one child. + let replies: Vec<_> = self + .children + .iter() + .map(|c| { + // A `ReplyReceiver` is `IntoFuture`; `join_all` takes futures. + std::future::IntoFuture::into_future( + c.send_request(msg.clone(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ) + }) + .collect(); + let mut nodes = 1; + let mut tag = self.id as u64 + 1; + let mut checksum = value; + for reply in futures::future::join_all(replies).await { + let ack = reply.unwrap(); + assert_eq!(ack.id, msg.id); + nodes += ack.nodes; + tag += ack.tag; + checksum ^= ack.checksum; + } + let completed = Instant::now(); + if msg.measured { + self.meter.completed.fetch_add(1, Ordering::Relaxed); + } + Ack { + id: msg.id, + nodes, + tag, + checksum, + completed, + queue_ns: started.duration_since(msg.enqueued).as_nanos() as u64, + service_ns: completed.duration_since(started).as_nanos() as u64, + } + }; + reply.reply(__reply_value); } } fn layout(c: &Config) -> (Vec, Vec) { @@ -352,7 +373,7 @@ struct Meta { } struct Pending { meta: Meta, - reply: tokio::sync::oneshot::Receiver, + reply: spawned_concurrency::ReplyReceiver, } fn sample( p: Meta, @@ -585,24 +606,31 @@ fn main() { runtime .block_on( nodes[id] - .request_raw(Wire { - children, - expected_shard: assignment[id], - }) - .unwrap(), + .send_request( + Wire { + children, + expected_shard: assignment[id], + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + .into_future(), ) .unwrap(); } for (id, node) in nodes.iter().enumerate() { let reply = node - .request_raw(Route { - id: id as u64, - enqueued: Instant::now(), - iterations: c.iterations, - measured: false, - }) + .send_request( + Route { + id: id as u64, + enqueued: Instant::now(), + iterations: c.iterations, + measured: false, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); - runtime.block_on(reply).unwrap(); + runtime.block_on(reply.into_future()).unwrap(); } let done = Arc::new(AtomicBool::new(false)); let observed = Arc::new(AtomicU64::new(0)); @@ -649,43 +677,45 @@ fn main() { epoch.changed().await.unwrap(); let start = epoch.borrow().unwrap(); wait_until(start); - let producers = (driver..c.producers) - .step_by(c.caller_threads) - .map(|producer| async move { - let mut samples = - Vec::with_capacity(c.messages.div_ceil(c.producers)); - for id in (producer..c.messages).step_by(c.producers) { - let intended = Instant::now(); - let enqueued = Instant::now(); - let (target, hot) = c.target(id); - let reply = nodes[target] - .request_raw(Route { + let producers = + (driver..c.producers).step_by(c.caller_threads).map( + |producer| async move { + let mut samples = Vec::with_capacity( + c.messages.div_ceil(c.producers), + ); + for id in (producer..c.messages).step_by(c.producers) { + let intended = Instant::now(); + let enqueued = Instant::now(); + let (target, hot) = c.target(id); + let reply = nodes[target] + .send_request(Route { id: id as u64, enqueued, iterations: c.iterations, measured: false, - }) + }, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) .unwrap(); - let ack = reply.await.unwrap(); - samples.push(sample( - Meta { - id, - target, - hot, - intended, - enqueued, - }, - Some(ack), - Instant::now(), - c, - assignment, - )); - if c.observation_counter { - observed.fetch_add(1, Ordering::Relaxed); + let ack = reply.await.unwrap(); + samples.push(sample( + Meta { + id, + target, + hot, + intended, + enqueued, + }, + Some(ack), + Instant::now(), + c, + assignment, + )); + if c.observation_counter { + observed.fetch_add(1, Ordering::Relaxed); + } } - } - samples - }); + samples + }, + ); futures::future::join_all(producers) .await .into_iter() @@ -774,12 +804,15 @@ fn main() { meters[target].submitted.fetch_add(1, Ordering::Relaxed); } let reply = nodes[target] - .request_raw(Route { - id: id as u64, - enqueued, - iterations, - measured: c.mode != "graph", - }) + .send_request( + Route { + id: id as u64, + enqueued, + iterations, + measured: c.mode != "graph", + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); let meta = Meta { id, @@ -791,7 +824,8 @@ fn main() { if let Some(tx) = &tx { tx.send(Pending { meta, reply }).unwrap(); } else { - let ack = futures::executor::block_on(reply).unwrap(); + let ack = + futures::executor::block_on(reply.into_future()).unwrap(); samples.push(sample( meta, Some(ack), @@ -844,11 +878,15 @@ fn main() { for (id, node) in nodes.iter().enumerate() { runtime .block_on( - node.request_raw(Wire { - children: Vec::new(), - expected_shard: assignment[id], - }) - .unwrap(), + node.send_request( + Wire { + children: Vec::new(), + expected_shard: assignment[id], + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + .into_future(), ) .unwrap(); } diff --git a/concurrency/examples/panic_abort.rs b/concurrency/examples/panic_abort.rs index 90b567981..efa00df89 100644 --- a/concurrency/examples/panic_abort.rs +++ b/concurrency/examples/panic_abort.rs @@ -43,8 +43,13 @@ mod native { } } impl Handler for Victim { - fn handle(&mut self, _: Boom, _: &Context) -> u64 { - panic!("{PANIC_TEXT}") + fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + panic!("{PANIC_TEXT}"); } } pub fn handler_panic() { @@ -66,8 +71,13 @@ mod tasks { } } impl Handler for Victim { - async fn handle(&mut self, _: Boom, _: &Context) -> u64 { - panic!("{PANIC_TEXT}") + async fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + panic!("{PANIC_TEXT}"); } } pub fn handler_panic() { @@ -93,15 +103,12 @@ mod tasks { tokio::runtime::Runtime::new().unwrap().block_on(async { let system = ActorSystem::new().unwrap(); let group = system.async_group("abort", 1).unwrap(); - let config = SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(1), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(1)) + .jitter(false), + ); let mut tree = system.supervisor(config).unwrap(); let child = tree .add( @@ -118,7 +125,8 @@ mod tasks { .await .expect("no restart after Kill") .expect("child unavailable after Kill"); - new.child_handle().shutdown(); + new.child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(new.wait_exit().await, ExitReason::Shutdown); let stayed = tokio::time::timeout(LIMIT, child.wait_for_generation(first + 1)).await; assert!( diff --git a/concurrency/src/actor_cleanup.rs b/concurrency/src/actor_cleanup.rs index 69e376ff7..5e7a1507a 100644 --- a/concurrency/src/actor_cleanup.rs +++ b/concurrency/src/actor_cleanup.rs @@ -1,8 +1,9 @@ //! Shared terminal publication for actor futures and native threads. +use crate::lifecycle_trace as trace; use crate::{ child_handle::ActorId, error::ExitReason, - link::{LinkCleanup, LinkTable, LinkedExitReason}, + link::{LinkNotices, LinkTable, LinkedExitReason}, monitor::{MonitorCleanup, MonitorTable}, }; use spawned_rt::sync::watch; @@ -14,19 +15,55 @@ pub(crate) fn log_safely(log: impl FnOnce()) { } } +/// `stopped()` did not run for this exit (decision 2: a kill, an untrapped +/// exit signal or link notice, or the runtime dropping the actor). Logged at +/// `debug` for every skip, whether or not the actor overrides `stopped()` +/// (there is no cheap way to know): an application whose cleanup must run +/// traps exits (`ctx.trap_exit(true)`) so a parent's shutdown runs it. +pub(crate) fn log_stopped_skipped( + actor: ActorId, + actor_type: &'static str, + cause: &'static str, + reason: Option<&ExitReason>, +) { + let reason = reason.cloned().unwrap_or(ExitReason::Killed); + log_safely(|| { + tracing::debug!(event = "stopped_skipped", actor = %actor, actor_type, cause, + reason = %reason, "stopped() skipped: the exit was not begun by the actor, \ + an administrative stop or its parent"); + }); +} + /// Constructed before dispatch, so even a never-polled actor has cleanup. -/// Completion owns its sender until links and outgoing monitors are closed. +/// [`die`](Self::die) is the death point D; dropping it is publication. +/// +/// D is one boundary, whatever drives it (the run loop, or the runtime +/// dropping the actor): exit pending for dropped replies, job admission +/// closed and every job cancelled, outgoing replies and exit signals fenced, +/// the final reason fixed with the link table closed and admission sealed, +/// and every open deferred reply answered. The caller then answers its +/// queued requests (the mode's mailbox discharge) and only then destroys +/// payloads and state; publication comes last, when this is dropped. pub(crate) struct ActorCleanup { id: ActorId, metrics: crate::instrumentation::ActorObservation, - links: LinkCleanup, + links: LinkTable, cause: LinkedExitReason, monitors: Option, completion: watch::Sender>, - fallback: ExitReason, reason: Option, + notices: Option, cancel: C, - mailbox_failures: Option>, + failures: Option>, + /// The reply registry, sealed with the final reason at D. + replies: Option>, + /// The owner's jobs: admission closes and they are cancelled at D. + jobs: Option>, + /// A runtime drop skipped `stopped()`: logged once every obligation is + /// answered (a subscriber is user code). + skip_log: Option, + /// [`discharged`](Self::discharged) ran. + discharged: bool, } impl ActorCleanup { pub(crate) fn new( @@ -35,140 +72,251 @@ impl ActorCleanup { cause: LinkedExitReason, monitors: MonitorTable, completion: watch::Sender>, - fallback: ExitReason, cancel: C, ) -> Self { Self { id, metrics: crate::instrumentation::ActorObservation::new(), - links: LinkCleanup::new(id, links), + links, cause, monitors: Some(MonitorCleanup(monitors)), completion, - fallback, reason: None, + notices: None, cancel, - mailbox_failures: None, + failures: None, + replies: None, + jobs: None, + skip_log: None, + discharged: false, } } + /// Seal `replies` with the final reason at D, inside the exit-table + /// critical section: callers then get `ActorStopped(final reason)` and + /// later sends `NoProc`. + pub(crate) fn with_replies( + mut self, + replies: std::sync::Arc, + ) -> Self { + self.replies = Some(replies); + self + } + pub(crate) fn with_jobs(mut self, jobs: std::sync::Arc) -> Self { + self.jobs = Some(jobs); + self + } pub(crate) fn with_mailbox_failures( mut self, failures: std::sync::Arc, ) -> Self { - self.mailbox_failures = Some(failures); + self.failures = Some(failures); self } - pub(crate) fn complete(&mut self, reason: ExitReason) { - self.complete_inner(reason, false); + /// The death point D: the requested reason becomes final and the link + /// table closes, taking one notice per link (delivered at publication). + /// Idempotent; returns the final reason. + pub(crate) fn die(&mut self) -> ExitReason { + self.reach_death(false) } - pub(crate) fn complete_secondary(&mut self, reason: ExitReason) { - self.complete_inner(reason, true); + + /// D reached because the runtime dropped the actor before it got there + /// (G1): `Killed` unless a non-`Normal` reason was already requested. + pub(crate) fn die_dropped(&mut self) -> ExitReason { + self.reach_death(true) } - fn complete_inner(&mut self, mut reason: ExitReason, secondary: bool) { - let mut cause = self.cause.lock().unwrap_or_else(|p| p.into_inner()); - if secondary { - if let Some(committed) = cause.as_ref().filter(|c| **c != ExitReason::Normal) { - reason = committed.clone(); - } + + fn reach_death(&mut self, dropped: bool) -> ExitReason { + if let Some(reason) = &self.reason { + return reason.clone(); } - if matches!(reason, ExitReason::Normal) { - if let Some(linked) = cause.as_ref() { - reason = linked.clone(); - } - if reason == ExitReason::Normal { - if let Some(message) = self.mailbox_failures.as_ref().and_then(|f| f.first()) { - reason = ExitReason::Panic(format!("queued message destructor: {message}")); + // Exit pending: dropped replies wait for D; jobs stop being admitted + // and are cancelled (the job join comes after publication). + if let Some(replies) = &self.replies { + replies.exiting(); + } + // From here until every caller is answered, the payloads framework + // code drops for this owner (a scheduled emit expiring, a job's + // refused outcome) are held, on whatever thread drops them. + if let Some(failures) = &self.failures { + failures.hold(); + } + if let Some(jobs) = &self.jobs { + jobs.cancel_all(); + } + // Fence outgoing replies and exit signals before the final reason is + // visible (outside any trace section: an in-flight signal may need + // one). Later ones are refused; those in flight finish first, so a + // signal sent before death precedes the death notices. + if let Some(replies) = &self.replies { + replies.fence(); + } + let id = self.id.raw(); + let mut skipped = false; + let reason = trace::life_section(id, || { + let mut request = crate::link::lock_request(&self.cause); + let reason = if dropped { + skipped = !request.hook_begun(); + request.runtime_drop() + } else { + request.die() + }; + drop(request); + trace::life_emit(id, || { + trace::LifeEvent::new(if dropped { "Drop" } else { "Die" }, id) + .x(trace::life_reason(&reason)) + }); + self.close(reason) + }); + if skipped { + self.skip_log = Some(reason.clone()); + } + fail_point!("actor.at_death"); + // Every open deferred reply (and every exit-pending one) is answered + // with the final reason, before any destructor runs. Their wake-ups + // wait until the mailbox's queued requests are answered too + // ([`discharged`](Self::discharged)): no waker runs between two + // answers. + if let Some(replies) = &self.replies { + replies.begin_discharge(); + replies.close(); + } + reason + } + + /// The owner's registry (none for an actor without replies). + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn replies(&self) -> Option<&crate::reply::registry::Registry> { + self.replies.as_deref() + } + + /// Answer a destroyed turn's own caller at the death point (its slot is + /// held by the turn's `PendingReply`, not listed in the registry). + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn answer_turn( + &self, + caller: std::sync::Arc, + ) { + if let Some(replies) = &self.replies { + replies.answer_turn(caller); + } + } + + /// Every obligation of the death point is answered (the mode answered + /// its queued requests after [`die`](Self::die)): now run the deferred + /// wake-ups, then log a skipped `stopped()`, then destroy what the jobs' + /// cancellation released. Call it before destroying the mailbox's + /// payloads and the state. Idempotent. + pub(crate) fn discharged(&mut self) { + if std::mem::replace(&mut self.discharged, true) { + return; + } + if let Some(replies) = &self.replies { + replies.end_discharge(); + } + if let Some(reason) = self.skip_log.take() { + log_stopped_skipped(self.id, "", "runtime_drop", Some(&reason)); + } + // Payloads the exit's job cancellation released: destroyed only now. + if let Some(jobs) = &self.jobs { + jobs.dispose_cancelled(); + } + if let Some(failures) = &self.failures { + failures.release(); + } + } + fn close(&mut self, reason: ExitReason) -> ExitReason { + let replies = self.replies.as_deref(); + self.notices = Some(crate::link::close_links_at_death( + self.id, + &self.links, + &reason, + |reason| { + if let Some(replies) = replies { + replies.seal(reason); } - } + }, + )); + self.reason = Some(reason.clone()); + reason + } + /// Record a cleanup failure after D (state or queued-envelope + /// destructor): it never changes the final reason. + pub(crate) fn cleanup_failed(&self, message: String) { + if let Some(failures) = &self.failures { + failures.record(message); } - // Freeze the completed outcome before any further cancellation callbacks. - *cause = Some(reason.clone()); - self.reason = Some(reason); } } impl Drop for ActorCleanup { fn drop(&mut self) { - let completed = self.reason.is_some(); - let failure = self.mailbox_failures.as_ref().and_then(|f| f.first()); - let mut reason = self.reason.take().unwrap_or_else(|| self.fallback.clone()); - { - let mut cause = self.cause.lock().unwrap_or_else(|p| p.into_inner()); - if !completed { - if let Some(committed) = cause.as_ref().filter(|c| **c != ExitReason::Normal) { - reason = committed.clone(); - } - } - // Queued destructors are secondary to committed Shutdown/Kill or - // a linked/handler panic. Only a completed Normal lifecycle gains - // a new panic outcome; aborted tasks keep their abort outcome. - if completed && reason == ExitReason::Normal { - if let Some(message) = &failure { - reason = ExitReason::Panic(format!("queued message destructor: {message}")); - } - } - *cause = Some(reason.clone()); - } - if let Some(message) = failure { + // Normally D was reached already; otherwise the runtime dropped the + // actor (G1) and nothing of it is left to destroy. + let reason = self.die_dropped(); + self.discharged(); + if let Some(message) = self.failures.as_ref().and_then(|f| f.first()) { // Subscriber code must not unwind through a receiver/actor teardown. - if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - tracing::warn!(event = "mailbox_cleanup_failed", actor = %self.id, %message, %reason, - "queued message destructor failed"); - })) { - crate::mailbox::dispose_panic(payload); - } + log_safely(|| { + tracing::warn!(event = "actor_cleanup_failed", actor = %self.id, %message, %reason, + "destructor failed after the death point"); + }); } (self.cancel)(); drop(self.monitors.take()); - self.links.complete(&reason); self.metrics.finish(&reason); + // Publication: the link notices taken at D are delivered, then + // wait_exit, join and monitors observe the final reason. Delivering + // first keeps v0.7's guarantee that once `wait_exit` returns, every + // linked peer's `Exit` is already queued (or its exit requested). + // The trace's `Publish` is this point: destruction is over and the + // notices are delivered after it. + let id = self.id.raw(); + trace::life_section(id, || { + trace::life_emit(id, || trace::LifeEvent::new("Publish", id)); + }); + if let Some(notices) = self.notices.take() { + notices.deliver(); + } self.completion.send_replace(Some(reason)); } } -/// Destroy state after capturing the lifecycle outcome. A state destructor is -/// secondary to an already observed handler/start/stop panic, even before that -/// outcome has been committed to the shared exit slot. -pub(crate) fn finish_actor_state( - actor: A, - primary: Result, -) -> Result { +/// Destroy state after D. A destructor panic never changes the final reason; +/// it is returned as a cleanup failure. +pub(crate) fn finish_actor_state(actor: A) -> Option { let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(actor))) else { - return primary; + return None; }; let message = crate::error::panic_message(&*payload); crate::mailbox::dispose_panic(payload); - let keep_primary = match &primary { - Ok(reason) => *reason != ExitReason::Normal, - Err(_) => true, - }; - if keep_primary { - if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - tracing::warn!(event = "actor_state_cleanup_failed", actor_type = std::any::type_name::(), - ?primary, secondary = %message, "actor state destructor panicked after a terminal outcome"); - })) { - crate::mailbox::dispose_panic(payload); - } - primary - } else { - Err(ExitReason::Panic(format!( - "actor state destructor: {message}" - ))) - } + log_safely(|| { + tracing::warn!(event = "actor_state_cleanup_failed", actor_type = std::any::type_name::(), + secondary = %message, "actor state destructor panicked after the death point"); + }); + Some(format!("actor state destructor: {message}")) } -/// Captured as one value by the dispatch closure. Declaration order makes -/// never-polled payload destruction precede completion publication, without -/// relying on the compiler's order for disjoint closure captures. +/// Captured as one value by the dispatch closure, so a never-polled actor +/// dropped with its runtime still reaches D, then answers its queued +/// requests (`discharge`), then destroys its payload, then publishes, +/// without relying on the compiler's order for disjoint closure captures. pub(crate) struct ActorStartup { payload: Option

, cleanup: Option>, + /// Answers the queued requests and returns what it drained, each item + /// destroyed on its own (contained) once the wake-ups ran. + discharge: fn(&mut P) -> Vec>, } impl ActorStartup { - pub(crate) fn new(payload: P, cleanup: ActorCleanup) -> Self { + pub(crate) fn new( + payload: P, + cleanup: ActorCleanup, + discharge: fn(&mut P) -> Vec>, + ) -> Self { Self { payload: Some(payload), cleanup: Some(cleanup), + discharge, } } pub(crate) fn into_parts(mut self) -> (P, ActorCleanup) { @@ -177,16 +325,36 @@ impl ActorStartup { } impl Drop for ActorStartup { fn drop(&mut self) { - if let Some(payload) = self.payload.take() { + if let Some(cleanup) = &mut self.cleanup { + cleanup.die_dropped(); + } + if let Some(mut payload) = self.payload.take() { + // Answers the queued requests; their payloads are destroyed with + // the rest of the startup payload, after the wake-ups. + let queued = (self.discharge)(&mut payload); + if let Some(cleanup) = &mut self.cleanup { + cleanup.discharged(); + } + // One at a time: a panicking destructor must not unwind into the + // next one (that would abort). + for item in queued { + if let Err(panic) = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(item))) + { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + if let Some(cleanup) = &mut self.cleanup { + cleanup.cleanup_failed(message); + } + } + } if let Err(panic) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| drop(payload))) { let message = crate::error::panic_message(&*panic); crate::mailbox::dispose_panic(panic); if let Some(cleanup) = &mut self.cleanup { - cleanup.complete_secondary(ExitReason::Panic(format!( - "actor startup payload destructor: {message}" - ))); + cleanup.cleanup_failed(format!("actor startup payload destructor: {message}")); } } } @@ -203,22 +371,11 @@ mod state_drop_tests { } } #[test] - fn destructor_failure_only_escalates_normal_even_without_a_committed_cause() { - for primary in [ - Ok(ExitReason::Normal), - Ok(ExitReason::Shutdown), - Ok(ExitReason::Kill), - Ok(ExitReason::Panic("handler failure".into())), - Err(ExitReason::Panic("lifecycle unwind".into())), - ] { - let expected = if primary == Ok(ExitReason::Normal) { - Err(ExitReason::Panic( - "actor state destructor: secondary state failure".into(), - )) - } else { - primary.clone() - }; - assert_eq!(finish_actor_state(Explodes, primary), expected); - } + fn destructor_failure_is_a_cleanup_failure_not_a_reason() { + assert_eq!( + finish_actor_state(Explodes), + Some("actor state destructor: secondary state failure".into()) + ); + assert_eq!(finish_actor_state(()), None); } } diff --git a/concurrency/src/child_handle.rs b/concurrency/src/child_handle.rs index dbcd8611f..3b94d2136 100644 --- a/concurrency/src/child_handle.rs +++ b/concurrency/src/child_handle.rs @@ -1,5 +1,5 @@ use crate::error::ExitReason; -use crate::link::{LinkTable, LinkedExitReason, SendExitFn, TrapExitFlag}; +use crate::link::{LinkTable, Origin, SignalTarget}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -15,6 +15,14 @@ static NEXT_ACTOR_ID: AtomicU64 = AtomicU64::new(1); pub struct ActorId(u64); impl ActorId { + /// The sender of an exit signal sent from outside any actor + /// ([`ChildHandle::exit`]). Never an actor's identity, never a parent. + pub const EXTERNAL: ActorId = ActorId(0); + + #[cfg_attr(not(feature = "lifecycle-trace"), allow(dead_code))] + pub(crate) fn raw(self) -> u64 { + self.0 + } pub(crate) fn next() -> Self { Self(NEXT_ACTOR_ID.fetch_add(1, Ordering::Relaxed)) } @@ -49,85 +57,55 @@ pub(crate) enum Completion { pub(crate) type CancelFn = Arc; /// Type-erased handle to a running actor. Provides lifecycle operations -/// (stop, liveness check, exit reason) without knowing the actor's concrete type. +/// (administrative stops, exit signals, liveness check, exit reason) without +/// knowing the actor's concrete type. /// /// Obtained via `ChildHandle::from(actor_ref)`. /// /// Unlike `ActorRef`, a `ChildHandle` cannot send messages directly โ€” it -/// only provides supervision-related operations (stop, wait, check liveness, -/// link/monitor). +/// only provides supervision-related operations (stop, exit, kill, wait, +/// check liveness, link/monitor). +/// +/// Ending an actor, as in OTP: +/// +/// | Call | Ends with | `stopped()` | Trapping actor | OTP | +/// |---|---|---|---|---| +/// | `stop()` | `Normal` | runs | ignores trapping | `gen_server:stop/1` | +/// | `stop_with_reason(r)` | `r` (also `Kill`) | runs | ignores trapping | `gen_server:stop/3` | +/// | `exit(r)` | `r`, or nothing for `Normal` | skipped | gets `Exit { from: EXTERNAL, reason: r }` | `exit(Pid, R)` | +/// | `kill()` = `exit(Kill)` | `Killed` | skipped (unless begun) | cannot trap it | `exit(Pid, kill)` | +/// +/// So `stop_with_reason(Kill)` ends `Kill` after `stopped()` and links see a +/// trappable `Kill`, while `kill()` ends `Killed`, untrappable. #[derive(Clone)] pub struct ChildHandle { - id: ActorId, + target: Arc, cancel: CancelFn, completion: Completion, - // Link/trap_exit support โ€” carried so `ctx.link()` can wire up signal - // propagation without knowing the underlying actor type. - trap_exit: TrapExitFlag, links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, mailbox_failures: Option>, } impl std::fmt::Debug for ChildHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("ChildHandle") - .field("id", &self.id) + .field("id", &self.target.id) .finish_non_exhaustive() } } impl ChildHandle { - /// Create a ChildHandle for tasks mode. - #[allow(clippy::too_many_arguments)] - #[cfg(feature = "async")] - pub(crate) fn from_tasks( - id: ActorId, - cancellation_token: spawned_rt::tasks::CancellationToken, - completion_rx: spawned_rt::sync::watch::Receiver>, - trap_exit: TrapExitFlag, - links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, - ) -> Self { - let stop_reason = linked_reason.clone(); + /// Create a ChildHandle from an actor's signal target, completion channel + /// and exit table. + pub(crate) fn new(target: Arc, completion: Completion, links: LinkTable) -> Self { + let stop = target.clone(); Self { - id, cancel: Arc::new(move || { - crate::link::normal_stop(&stop_reason, || cancellation_token.cancel()) + crate::link::normal_stop(stop.id, &stop.request, || (stop.wake)()) }), - completion: Completion::Tasks(completion_rx), - trap_exit, - links, - linked_reason, - send_exit, - mailbox_failures: None, - } - } - - /// Create a ChildHandle for threads mode. - #[allow(clippy::too_many_arguments)] - pub(crate) fn from_threads( - id: ActorId, - cancellation_token: spawned_rt::threads::CancellationToken, - completion: spawned_rt::sync::watch::Receiver>, - trap_exit: TrapExitFlag, - links: LinkTable, - linked_reason: LinkedExitReason, - send_exit: SendExitFn, - ) -> Self { - let stop_reason = linked_reason.clone(); - Self { - id, - cancel: Arc::new(move || { - crate::link::normal_stop(&stop_reason, || cancellation_token.cancel()) - }), - completion: Completion::Threads(completion), - trap_exit, + target, + completion, links, - linked_reason, - send_exit, mailbox_failures: None, } } @@ -139,28 +117,34 @@ impl ChildHandle { self } - /// First queued-message destructor failure, recorded independently of the - /// primary exit reason. Complete after actor termination; Shutdown, Kill - /// and an existing panic are never replaced by this secondary failure. - pub fn mailbox_cleanup_failure(&self) -> Option { + /// First cleanup failure after the death point (a panicking state or + /// queued-message destructor), recorded independently of the final exit + /// reason, which it never changes. Complete after publication. A + /// supervisor restarts a Transient child that has one. + pub fn cleanup_failure(&self) -> Option { self.mailbox_failures.as_ref().and_then(|f| f.first()) } - /// Crate-internal accessors used by `ctx.link()` to wire up signal propagation. - pub(crate) fn trap_exit_flag(&self) -> &TrapExitFlag { - &self.trap_exit + /// Deprecated alias of [`cleanup_failure`](Self::cleanup_failure). + #[deprecated(since = "0.8.0", note = "use `cleanup_failure()`")] + pub fn mailbox_cleanup_failure(&self) -> Option { + self.cleanup_failure() + } + + /// Crate-internal accessors used by `ctx.link()` and `ctx.monitor()`. + pub(crate) fn signal_target(&self) -> &Arc { + &self.target } pub(crate) fn links(&self) -> &LinkTable { &self.links } - pub(crate) fn linked_reason(&self) -> &LinkedExitReason { - &self.linked_reason - } - pub(crate) fn cancel_fn(&self) -> &CancelFn { - &self.cancel - } - pub(crate) fn send_exit_fn(&self) -> &SendExitFn { - &self.send_exit + /// Past the death point D: links, monitors, sends and requests report + /// `NoProc` from now on. + pub(crate) fn is_past_death(&self) -> bool { + self.links + .lock() + .unwrap_or_else(|p| p.into_inner()) + .is_dead() } // Each child_handle() creates this allocation; its clones share it. @@ -171,28 +155,55 @@ impl ChildHandle { /// The actor's unique identity. pub fn id(&self) -> ActorId { - self.id + self.target.id } - /// Signal the actor to stop. The actor will finish its current handler, - /// run `stopped()`, and exit. + /// Administrative stop (Spawned's `gen_server:stop`): the actor finishes + /// its current handler, discards its queue, runs `stopped()` and exits + /// `Normal`. Untrappable; returns at once (join with `wait_exit_*`). pub fn stop(&self) { (self.cancel)(); } - /// Direct cooperative stop with a chosen reason. Trapping only affects - /// linked exit notices and cannot intercept this request. Kill remains - /// trappable when this actor's final reason is propagated to its peers. + /// Administrative stop with a chosen reason, as [`stop`](Self::stop). + /// Not an exit signal: `trap_exit` cannot intercept it and `stopped()` + /// runs. `stop_with_reason(Kill)` ends `Kill`; use [`kill`](Self::kill) + /// for the untrappable kill signal that ends `Killed`. pub fn stop_with_reason(&self, reason: ExitReason) { - crate::link::stop_with_reason(&self.linked_reason, reason, || (self.cancel)()); + crate::link::stop_with_reason(self.target.id, &self.target.request, reason, || { + (self.target.wake)() + }); } - /// Request an ordered cooperative shutdown without replacing an earlier - /// committed cause. A cleanup panic remains an abnormal exit. - pub fn shutdown(&self) { - crate::link::stop_with_reason(&self.linked_reason, ExitReason::Shutdown, || { - (self.cancel)() - }); + /// Send an exit signal from outside any actor (OTP `exit_signal/2`, the + /// sender is [`ActorId::EXTERNAL`]): + /// + /// - `Kill` is untrappable: the actor ends `Killed` (see [`kill`](Self::kill)); + /// - `Normal` is ignored by a non-trapping actor; + /// - any other reason ends a non-trapping actor with that reason after + /// its current handler, **without** `stopped()`, discarding its queue; + /// - a trapping actor gets `Exit { from: EXTERNAL, reason }` queued + /// behind its messages. + /// + /// A signal to an actor past its death point is dropped. + pub fn exit(&self, reason: ExitReason) { + self.exit_from(ActorId::EXTERNAL, reason, false); + } + + /// Untrappable kill signal (`exit(Kill)`): the actor ends `Killed`, also + /// when it is already exiting (until its death point), and nothing + /// replaces that reason. The current handler, and a `stopped()` that + /// already began, still finish (Rust cannot preempt them); a `stopped()` + /// that has not begun is skipped. Links receive `Killed` (trappable), + /// monitors `Down { reason: Killed }`. + pub fn kill(&self) { + self.exit(ExitReason::Kill); + } + + /// Exit signal with an explicit sender; `from_parent` marks the internal + /// entry point of supervisors and systems (E3). + pub(crate) fn exit_from(&self, from: ActorId, reason: ExitReason, from_parent: bool) { + crate::link::deliver_signal(&self.target, from, reason, Origin::Direct, from_parent); } /// Returns `true` if the actor is still running. @@ -200,9 +211,10 @@ impl ChildHandle { self.exit_reason().is_none() } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods (for example, after + /// runtime shutdown). pub fn exit_reason(&self) -> Option { match &self.completion { #[cfg(feature = "async")] @@ -256,18 +268,20 @@ pub(crate) fn poll_exit_reason( let closed = rx.has_changed().is_err(); rx.borrow() .clone() - .or_else(|| closed.then_some(ExitReason::Kill)) + .or_else(|| closed.then_some(ExitReason::Killed)) } /// Async loop that polls a watch channel until it carries an exit reason. -/// Returns `ExitReason::Kill` if the watch sender is dropped without a reason. -async fn wait_loop(mut rx: spawned_rt::sync::watch::Receiver>) -> ExitReason { +/// Returns `ExitReason::Killed` if the watch sender is dropped without a reason. +pub(crate) async fn wait_loop( + mut rx: spawned_rt::sync::watch::Receiver>, +) -> ExitReason { loop { if let Some(reason) = rx.borrow_and_update().clone() { return reason; } if rx.changed().await.is_err() { - return ExitReason::Kill; + return ExitReason::Killed; } } } @@ -308,7 +322,7 @@ pub(crate) fn wait_for_thread_exit_blocking( /// making progress (deadlock). Use `wait_exit_async` from async context, or /// run on a multi-thread runtime. /// -/// Returns `ExitReason::Kill` if the watch sender is dropped without setting a +/// Returns `ExitReason::Killed` if the watch sender is dropped without setting a /// reason โ€” this means the actor task was aborted externally (e.g., runtime /// shutdown) without going through the normal exit path. #[cfg(feature = "async")] @@ -346,7 +360,7 @@ fn wait_for_tasks_exit_blocking( impl PartialEq for ChildHandle { fn eq(&self, other: &Self) -> bool { - self.id == other.id + self.target.id == other.target.id } } @@ -354,15 +368,32 @@ impl Eq for ChildHandle {} impl std::hash::Hash for ChildHandle { fn hash(&self, state: &mut H) { - self.id.hash(state); + self.target.id.hash(state); } } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; + /// A signal target whose wake cancels `token`. + pub(crate) fn test_target( + id: ActorId, + token: spawned_rt::threads::CancellationToken, + ) -> Arc { + Arc::new(SignalTarget { + id, + parent: None, + trap: new_trap_exit_flag(), + request: new_linked_exit_reason(), + wake: Arc::new(move || token.cancel()), + send_exit: Arc::new(|_| Ok(())), + send_parent_exit: Arc::new(|_| Ok(())), + send_held: Arc::new(|_, _| Ok(())), + }) + } + // Shared tasks-mode fixture: an Idler actor that does nothing and never // stops on its own. Used by multiple tests below. #[cfg(feature = "async")] @@ -378,15 +409,10 @@ mod tests { token: spawned_rt::threads::CancellationToken, completion: spawned_rt::sync::watch::Receiver>, ) -> ChildHandle { - let no_op_send_exit: SendExitFn = Arc::new(|_| Ok(())); - ChildHandle::from_threads( - ActorId::next(), - token, - completion, - new_trap_exit_flag(), + ChildHandle::new( + test_target(ActorId::next(), token), + Completion::Threads(completion), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit, ) } @@ -436,24 +462,15 @@ mod tests { let (_completion_tx, completion) = spawned_rt::sync::watch::channel(None); let token = spawned_rt::threads::CancellationToken::new(); let id = ActorId::next(); - let no_op_send_exit: SendExitFn = Arc::new(|_| Ok(())); - let h1 = ChildHandle::from_threads( - id, - token.clone(), - completion.clone(), - new_trap_exit_flag(), + let h1 = ChildHandle::new( + test_target(id, token.clone()), + Completion::Threads(completion.clone()), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit.clone(), ); - let h2 = ChildHandle::from_threads( - id, - token, - completion, - new_trap_exit_flag(), + let h2 = ChildHandle::new( + test_target(id, token), + Completion::Threads(completion), new_link_table(), - new_linked_exit_reason(), - no_op_send_exit, ); assert_eq!(h1, h2); } @@ -462,306 +479,390 @@ mod tests { #[test] fn child_handle_from_threads_actor_ref() { - use crate::message::Message; - use crate::threads::actor::{Actor, ActorStart, Context, Handler}; - - struct Worker; - struct Stop; - impl Message for Stop { - type Result = (); - } - impl Actor for Worker {} - impl Handler for Worker { - fn handle(&mut self, _msg: Stop, ctx: &Context) { - ctx.stop(); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_from_threads_actor_ref"), + move || { + use crate::message::Message; + use crate::threads::actor::{Actor, ActorStart, Context, Handler}; + + struct Worker; + struct Stop; + impl Message for Stop { + type Result = (); + } + impl Actor for Worker {} + impl Handler for Worker { + fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = Worker.start(); - let handle = actor.child_handle(); + let actor = Worker.start(); + let handle = actor.child_handle(); - assert!(handle.is_alive()); - assert!(handle.exit_reason().is_none()); - assert_eq!(handle.id(), actor.id()); + assert!(handle.is_alive()); + assert!(handle.exit_reason().is_none()); + assert_eq!(handle.id(), actor.id()); - actor.send(Stop).unwrap(); - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - assert!(!handle.is_alive()); + actor.send(Stop).unwrap(); + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + assert!(!handle.is_alive()); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_from_tasks_actor_ref() { - use crate::message::Message; - use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; - - struct Worker; - struct Stop; - impl Message for Stop { - type Result = (); - } - impl Actor for Worker {} - impl Handler for Worker { - async fn handle(&mut self, _msg: Stop, ctx: &Context) { - ctx.stop(); - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Worker.start(); - let handle = actor.child_handle(); - - assert!(handle.is_alive()); - assert!(handle.exit_reason().is_none()); - assert_eq!(handle.id(), actor.id()); + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_from_tasks_actor_ref"), + move || { + use crate::message::Message; + use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; + + struct Worker; + struct Stop; + impl Message for Stop { + type Result = (); + } + impl Actor for Worker {} + impl Handler for Worker { + async fn handle( + &mut self, + _msg: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - actor.send(Stop).unwrap(); - let reason = handle.wait_exit_async().await; - assert_eq!(reason, ExitReason::Normal); - assert!(!handle.is_alive()); - }); + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Worker.start(); + let handle = actor.child_handle(); + + assert!(handle.is_alive()); + assert!(handle.exit_reason().is_none()); + assert_eq!(handle.id(), actor.id()); + + actor.send(Stop).unwrap(); + let reason = handle.wait_exit_async().await; + assert_eq!(reason, ExitReason::Normal); + assert!(!handle.is_alive()); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_stop_from_tasks_actor() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - - assert!(handle.is_alive()); - handle.stop(); - let reason = handle.wait_exit_async().await; - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::child_handle_stop_from_tasks_actor"), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + + assert!(handle.is_alive()); + handle.stop(); + let reason = handle.wait_exit_async().await; + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_wait_blocking_inside_multithread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - // Multi-thread runtime โ€” wait_exit_blocking should work via block_in_place - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - handle.stop(); - - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_inside_multithread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + // Multi-thread runtime โ€” wait_exit_blocking should work via block_in_place + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + handle.stop(); + + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } #[test] #[should_panic(expected = "non-multi-thread tokio runtime")] #[cfg(feature = "async")] fn child_handle_wait_blocking_panics_on_current_thread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = ::tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - // Calling wait_exit_blocking from within a current-thread runtime - // would deadlock the actor task โ€” we panic with a clear message instead. - let _ = handle.wait_exit_blocking(); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_panics_on_current_thread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = ::tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + // Calling wait_exit_blocking from within a current-thread runtime + // would deadlock the actor task โ€” we panic with a clear message instead. + let _ = handle.wait_exit_blocking(); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_wait_blocking_fast_path_on_current_thread_runtime() { - use crate::tasks::actor::ActorStart; - use tasks_fixtures::Idler; - - let runtime = ::tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - // Fast path: if the actor has already exited, wait_exit_blocking is safe - // even on a current-thread runtime (no actual blocking happens). - runtime.block_on(async { - let actor = Idler.start(); - let handle = actor.child_handle(); - handle.stop(); - // Wait async first so the actor actually exits - let _ = handle.wait_exit_async().await; - // Now wait_exit_blocking takes the fast path - let reason = handle.wait_exit_blocking(); - assert_eq!(reason, ExitReason::Normal); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_wait_blocking_fast_path_on_current_thread_runtime" + ), + move || { + use crate::tasks::actor::ActorStart; + use tasks_fixtures::Idler; + + let runtime = ::tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + + // Fast path: if the actor has already exited, wait_exit_blocking is safe + // even on a current-thread runtime (no actual blocking happens). + runtime.block_on(async { + let actor = Idler.start(); + let handle = actor.child_handle(); + handle.stop(); + // Wait async first so the actor actually exits + let _ = handle.wait_exit_async().await; + // Now wait_exit_blocking takes the fast path + let reason = handle.wait_exit_blocking(); + assert_eq!(reason, ExitReason::Normal); + }); + }, + ); } // --- Panic observation tests --- #[test] fn child_handle_observes_panic_in_threads_actor() { - use crate::message::Message; - use crate::threads::actor::{Actor, ActorStart, Context, Handler}; - - struct Boomer; - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Actor for Boomer {} - impl Handler for Boomer { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { - panic!("intentional panic in handler"); - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_threads_actor" + ), + move || { + use crate::message::Message; + use crate::threads::actor::{Actor, ActorStart, Context, Handler}; + + struct Boomer; + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Actor for Boomer {} + impl Handler for Boomer { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic in handler"); + } + } + } - let actor = Boomer.start(); - let handle = actor.child_handle(); - let _ = actor.send(Boom); + let actor = Boomer.start(); + let handle = actor.child_handle(); + let _ = actor.send(Boom); - let reason = handle.wait_exit_blocking(); - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in handler"), "got: {msg}"); - } - other => panic!("expected Panic, got {other:?}"), - } - assert!(!handle.is_alive()); + let reason = handle.wait_exit_blocking(); + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + assert!(!handle.is_alive()); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_observes_panic_in_tasks_actor() { - use crate::message::Message; - use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; - - struct Boomer; - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Actor for Boomer {} - impl Handler for Boomer { - async fn handle(&mut self, _msg: Boom, _ctx: &Context) { - panic!("intentional panic in handler"); - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = Boomer.start(); - let handle = actor.child_handle(); - let _ = actor.send(Boom); - - let reason = handle.wait_exit_async().await; - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_tasks_actor" + ), + move || { + use crate::message::Message; + use crate::tasks::actor::{Actor, ActorStart, Context, Handler}; + + struct Boomer; + struct Boom; + impl Message for Boom { + type Result = (); } - other => panic!("expected Panic, got {other:?}"), - } - assert!(!handle.is_alive()); - }); + impl Actor for Boomer {} + impl Handler for Boomer { + async fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic in handler"); + } + } + } + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = Boomer.start(); + let handle = actor.child_handle(); + let _ = actor.send(Boom); + + let reason = handle.wait_exit_async().await; + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in handler"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + assert!(!handle.is_alive()); + }); + }, + ); } #[test] #[cfg(feature = "async")] fn child_handle_observes_panic_in_started_callback() { - use crate::tasks::actor::{Actor, ActorStart, Context}; - - struct PanicStart; - impl Actor for PanicStart { - async fn started(&mut self, _ctx: &Context) { - panic!("intentional panic in started"); - } - } - - let runtime = spawned_rt::tasks::Runtime::new().unwrap(); - runtime.block_on(async { - let actor = PanicStart.start(); - let handle = actor.child_handle(); - - let reason = handle.wait_exit_async().await; - match reason { - ExitReason::Panic(msg) => { - assert!(msg.contains("intentional panic in started"), "got: {msg}"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::child_handle_observes_panic_in_started_callback" + ), + move || { + use crate::tasks::actor::{Actor, ActorStart, Context}; + + struct PanicStart; + impl Actor for PanicStart { + async fn started(&mut self, _ctx: &Context) { + panic!("intentional panic in started"); + } } - other => panic!("expected Panic, got {other:?}"), - } - }); + + let runtime = spawned_rt::tasks::Runtime::new().unwrap(); + runtime.block_on(async { + let actor = PanicStart.start(); + let handle = actor.child_handle(); + + let reason = handle.wait_exit_async().await; + match reason { + ExitReason::Panic(msg) => { + assert!(msg.contains("intentional panic in started"), "got: {msg}"); + } + other => panic!("expected Panic, got {other:?}"), + } + }); + }, + ); } } #[cfg(test)] mod closed_completion_tests { use super::*; - use crate::link::{new_link_table, new_linked_exit_reason, new_trap_exit_flag}; + use crate::link::new_link_table; #[test] fn closed_without_reason_is_dead_and_all_waits_agree() { - #[cfg(not(feature = "async"))] - let modes = [true]; - #[cfg(feature = "async")] - let modes = [false, true]; - for threads in modes { - let (tx, rx) = spawned_rt::sync::watch::channel(None); - let child = if threads { - ChildHandle::from_threads( - ActorId::next(), - spawned_rt::threads::CancellationToken::new(), - rx, - new_trap_exit_flag(), - new_link_table(), - new_linked_exit_reason(), - Arc::new(|_| Ok(())), - ) - } else { + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_without_reason_is_dead_and_all_waits_agree" + ), + move || { #[cfg(not(feature = "async"))] - unreachable!(); + let modes = [true]; #[cfg(feature = "async")] - { - ChildHandle::from_tasks( + let modes = [false, true]; + for threads in modes { + let (tx, rx) = spawned_rt::sync::watch::channel(None); + let target = super::tests::test_target( ActorId::next(), - spawned_rt::tasks::CancellationToken::new(), - rx, - new_trap_exit_flag(), - new_link_table(), - new_linked_exit_reason(), - Arc::new(|_| Ok(())), - ) + spawned_rt::threads::CancellationToken::new(), + ); + let completion = if threads { + Completion::Threads(rx) + } else { + #[cfg(not(feature = "async"))] + unreachable!(); + #[cfg(feature = "async")] + Completion::Tasks(rx) + }; + let child = ChildHandle::new(target, completion, new_link_table()); + assert!(child.is_alive()); + drop(tx); + assert_eq!(child.exit_reason(), Some(ExitReason::Killed)); + assert!(!child.is_alive()); + assert_eq!( + futures::executor::block_on(child.wait_exit_async()), + ExitReason::Killed + ); + // Already-exited fast path must not enter another executor or reject + // a current-thread runtime merely because no exit value was published. + assert_eq!( + futures::executor::block_on(async { child.wait_exit_blocking() }), + ExitReason::Killed + ); + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + assert_eq!(child.wait_exit_blocking(), ExitReason::Killed); + }); } - }; - assert!(child.is_alive()); - drop(tx); - assert_eq!(child.exit_reason(), Some(ExitReason::Kill)); - assert!(!child.is_alive()); - assert_eq!( - futures::executor::block_on(child.wait_exit_async()), - ExitReason::Kill - ); - // Already-exited fast path must not enter another executor or reject - // a current-thread runtime merely because no exit value was published. - assert_eq!( - futures::executor::block_on(async { child.wait_exit_blocking() }), - ExitReason::Kill - ); - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - rt.block_on(async { - assert_eq!(child.wait_exit_blocking(), ExitReason::Kill); - }); - } + }, + ); } #[test] fn closure_preserves_the_final_published_reason() { diff --git a/concurrency/src/error.rs b/concurrency/src/error.rs index 87d08996e..61b622536 100644 --- a/concurrency/src/error.rs +++ b/concurrency/src/error.rs @@ -1,5 +1,9 @@ /// Reason an actor stopped. Used by supervision to decide whether to restart. +/// +/// Non-exhaustive, as OTP exit reasons are open terms: match with a wildcard +/// arm. #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub enum ExitReason { /// Clean stop via `ctx.stop()` or `ChildHandle::stop()`. The run loop holds /// a sender to its own mailbox, so the mailbox never closes while it runs. @@ -8,21 +12,40 @@ pub enum ExitReason { Shutdown, /// Actor panicked in `started()`, a handler, or `stopped()`. Panic(String), - /// Direct kill outcome, also used when an async task is aborted externally. - /// Propagated link notices carrying Kill are trappable. An active handler may finish and the - /// actor still runs `stopped()`. An externally dropped future cannot guarantee - /// execution of that async cleanup hook. No blocking Rust code is preempted. + /// A self-exit with `Kill` (`ctx.stop_with_reason(Kill)`, OTP `exit(kill)`), + /// an administrative `stop_with_reason(Kill)`, or a `Kill` link notice. + /// Propagated over links as `Kill` (trappable), not converted to `Killed`. Kill, + /// Killed by an untrappable kill signal ([`ChildHandle::kill`], + /// `exit(Kill)`), or dropped by its runtime before it finished (OTP + /// `killed`). Links receive `Killed`, which is trappable. + /// + /// [`ChildHandle::kill`]: crate::ChildHandle::kill + Killed, + /// The target no longer existed when a link, monitor, send or request + /// was made (OTP `noproc`). + NoProc, } impl ExitReason { /// Returns `true` for exit reasons that should trigger a restart - /// of `Transient` or `Permanent` children. - /// `Shutdown` is considered normal (no restart) matching Erlang semantics. + /// of `Transient` or `Permanent` children, and that end a non-trapping + /// linked actor (OTP: every reason except `normal` and `shutdown`). + /// + /// | Reason | `is_abnormal()` | + /// |---|---| + /// | `Normal` | `false` | + /// | `Shutdown` | `false` (an ordered stop, as in Erlang) | + /// | `Panic(_)` | `true` | + /// | `Kill` | `true` | + /// | `Killed` | `true` (OTP `killed`: a kill, a supervisor's kill escalation or a runtime drop) | + /// | `NoProc` | `true` (OTP `noproc`: the target was already gone) | pub fn is_abnormal(&self) -> bool { match self { ExitReason::Normal | ExitReason::Shutdown => false, - ExitReason::Panic(_) | ExitReason::Kill => true, + ExitReason::Panic(_) | ExitReason::Kill | ExitReason::Killed | ExitReason::NoProc => { + true + } } } } @@ -34,10 +57,18 @@ impl std::fmt::Display for ExitReason { ExitReason::Shutdown => write!(f, "shutdown"), ExitReason::Panic(msg) => write!(f, "panic: {msg}"), ExitReason::Kill => write!(f, "kill"), + ExitReason::Killed => write!(f, "killed"), + ExitReason::NoProc => write!(f, "noproc"), } } } +/// The target of `Context::try_link` no longer +/// existed (it was past its death point), as OTP's `noproc`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("noproc")] +pub struct NoProc; + /// Extract a human-readable message from a panic payload. pub(crate) fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { if let Some(s) = panic.downcast_ref::<&str>() { @@ -50,31 +81,105 @@ pub(crate) fn panic_message(panic: &(dyn std::any::Any + Send)) -> String { } /// Errors that can occur when communicating with an actor. +/// +/// Non-exhaustive: later releases may add errors. Match with a wildcard arm. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum ActorError { - /// The actor has stopped or its mailbox channel is closed. + /// The actor died before answering, with its final exit reason (OTP: + /// `gen_server:call` exits with `{Reason, {gen_server, call, ..}}`). /// - /// Returned by `send()` and `request()` when the actor is no longer running. - #[error("Actor stopped")] - ActorStopped, + /// A request that was queued, deferred or being handled when the actor + /// died is answered at its death point with the reason that `wait_exit`, + /// `Down` and linked actors see: `Killed` after a kill, the signal's + /// reason after an untrapped exit signal, `Panic(..)` after a crash. + #[error("Actor stopped: {0}")] + ActorStopped(ExitReason), + /// The actor no longer existed when the message or request was sent: it + /// was past its death point (OTP `noproc`). A send or request made while + /// the actor is still exiting is admitted instead, and a request is then + /// answered `ActorStopped(reason)` at its death. + #[error("noproc")] + NoProc, /// A request exceeded the timeout duration (default: 5 seconds). /// /// Returned by `request()` and `request_with_timeout()` when the actor /// does not reply in time. #[error("Request to Actor timed out")] RequestTimeout, + /// The handler dropped the request's reply obligation (a `PendingReply` + /// or a deferred `ReplyTo`) without answering it. + #[error("Reply dropped by the actor")] + ReplyDropped, + /// The actor rejected the request at a bound (for example its deferred + /// replies are at `MAX_OUTSTANDING`). + #[error("Actor overloaded")] + Overloaded, +} + +/// Classification helpers, so callers need not match every variant (the enum +/// is `#[non_exhaustive]`; a variant added later is classified here). +/// +/// | Variant | `target_gone` | `is_timeout` | `is_transient` | `exit_reason` | +/// |---|---|---|---|---| +/// | `ActorStopped(r)` | `true` | `false` | `false` | `Some(r)` | +/// | `NoProc` | `true` | `false` | `false` | `None` | +/// | `RequestTimeout` | `false` | `true` | `true` | `None` | +/// | `ReplyDropped` | `false` | `false` | `false` | `None` | +/// | `Overloaded` | `false` | `false` | `true` | `None` | +impl ActorError { + /// The target actor is gone: it died before answering + /// (`ActorStopped`) or was already past its death point (`NoProc`). + /// Retrying the same `ActorRef` cannot succeed; look the actor up again + /// (a supervisor may have restarted it under a new reference). + pub fn target_gone(&self) -> bool { + matches!(self, ActorError::ActorStopped(_) | ActorError::NoProc) + } + + /// The request was not answered within its timeout + /// (`RequestTimeout`). The actor may still be running and may still + /// handle the request: a timeout does not cancel it. + pub fn is_timeout(&self) -> bool { + matches!(self, ActorError::RequestTimeout) + } + + /// Retrying the same target may succeed: the actor was alive but did + /// not answer in time (`RequestTimeout`) or rejected the request at a + /// bound (`Overloaded`). `false` for a gone target, and for + /// `ReplyDropped`, where the actor is alive but its handler chose not to + /// answer (a decision or a bug, not a load condition): a retry would + /// usually get the same result. OTP has no such outcome (a `call` the + /// server never answers times out); the variant reports a drop at once + /// so a caller does not wait for its deadline, without claiming that the + /// same request will succeed later. A handler that means "not now, try + /// again" answers `reply.overloaded()` (or `token.overloaded()`), which is + /// transient; a caller whose protocol treats a drop as retryable matches + /// `ReplyDropped` explicitly. + pub fn is_transient(&self) -> bool { + matches!(self, ActorError::RequestTimeout | ActorError::Overloaded) + } + + /// The final exit reason of an actor that died before answering + /// (`ActorStopped`). `None` otherwise, including `NoProc` (the target + /// was already gone; its reason is not known to the caller). + pub fn exit_reason(&self) -> Option<&ExitReason> { + match self { + ActorError::ActorStopped(reason) => Some(reason), + _ => None, + } + } } impl From> for ActorError { fn from(_value: spawned_rt::threads::mpsc::SendError) -> Self { - Self::ActorStopped + Self::NoProc } } #[cfg(feature = "async")] impl From> for ActorError { fn from(_value: spawned_rt::tasks::mpsc::SendError) -> Self { - Self::ActorStopped + Self::NoProc } } @@ -84,16 +189,98 @@ mod tests { #[test] fn test_error_into_std_error() { - let error: &dyn std::error::Error = &ActorError::ActorStopped; - assert_eq!(error.to_string(), "Actor stopped"); + let error: &dyn std::error::Error = &ActorError::ActorStopped(ExitReason::Killed); + assert_eq!(error.to_string(), "Actor stopped: killed"); + assert_eq!(ActorError::NoProc.to_string(), "noproc"); + } + + /// Every `ExitReason`; the exhaustive match (allowed inside the crate) + /// makes adding a variant fail to compile until it is listed here. + fn all_exit_reasons() -> Vec { + let all = vec![ + ExitReason::Normal, + ExitReason::Shutdown, + ExitReason::Panic("boom".into()), + ExitReason::Kill, + ExitReason::Killed, + ExitReason::NoProc, + ]; + for reason in &all { + match reason { + ExitReason::Normal + | ExitReason::Shutdown + | ExitReason::Panic(_) + | ExitReason::Kill + | ExitReason::Killed + | ExitReason::NoProc => {} + } + } + all + } + + /// Every `ActorError`, with the same compile-time completeness check. + fn all_actor_errors() -> Vec { + let all = vec![ + ActorError::ActorStopped(ExitReason::Killed), + ActorError::NoProc, + ActorError::RequestTimeout, + ActorError::ReplyDropped, + ActorError::Overloaded, + ]; + for error in &all { + match error { + ActorError::ActorStopped(_) + | ActorError::NoProc + | ActorError::RequestTimeout + | ActorError::ReplyDropped + | ActorError::Overloaded => {} + } + } + all } #[test] fn exit_reason_is_abnormal_classification() { - assert!(!ExitReason::Normal.is_abnormal()); - assert!(!ExitReason::Shutdown.is_abnormal()); - assert!(ExitReason::Panic("boom".into()).is_abnormal()); - assert!(ExitReason::Kill.is_abnormal()); + for reason in all_exit_reasons() { + let expected = match reason { + ExitReason::Normal | ExitReason::Shutdown => false, + ExitReason::Panic(_) + | ExitReason::Kill + | ExitReason::Killed + | ExitReason::NoProc => true, + }; + assert_eq!(reason.is_abnormal(), expected, "{reason}"); + } + } + + #[test] + fn actor_error_classification_covers_every_variant() { + for error in all_actor_errors() { + // (target_gone, is_timeout, is_transient, exit_reason) + let expected = match &error { + ActorError::ActorStopped(r) => (true, false, false, Some(r.clone())), + ActorError::NoProc => (true, false, false, None), + ActorError::RequestTimeout => (false, true, true, None), + ActorError::ReplyDropped => (false, false, false, None), + ActorError::Overloaded => (false, false, true, None), + }; + let actual = ( + error.target_gone(), + error.is_timeout(), + error.is_transient(), + error.exit_reason().cloned(), + ); + assert_eq!(actual, expected, "{error:?}"); + } + } + + #[test] + fn actor_stopped_carries_every_reason() { + for reason in all_exit_reasons() { + let error = ActorError::ActorStopped(reason.clone()); + assert!(error.target_gone() && !error.is_transient() && !error.is_timeout()); + assert_eq!(error.exit_reason(), Some(&reason)); + } } #[test] @@ -102,6 +289,8 @@ mod tests { assert_eq!(ExitReason::Shutdown.to_string(), "shutdown"); assert_eq!(ExitReason::Panic("oops".into()).to_string(), "panic: oops"); assert_eq!(ExitReason::Kill.to_string(), "kill"); + assert_eq!(ExitReason::Killed.to_string(), "killed"); + assert_eq!(ExitReason::NoProc.to_string(), "noproc"); } #[test] diff --git a/concurrency/src/failpoints.rs b/concurrency/src/failpoints.rs index 5ff3f5867..ba32a6dd7 100644 --- a/concurrency/src/failpoints.rs +++ b/concurrency/src/failpoints.rs @@ -71,6 +71,44 @@ pub const SITES: &[Site] = &[ Delay models slow cleanup (shutdown deadlines); panic models a \ failing `stopped()`.", }, + Site { + name: "job.emit_due", + has_error: false, + description: "A scheduled `emit_at` comes due on the deadline service. A \ + handshake: one coming due during the owner's exit (`stopped()`) is \ + refused, and its payload is destroyed only after every caller is \ + answered (tasks and threads).", + }, + Site { + name: "job.native_worker_unused", + has_error: false, + description: "Threads mode: the parked thread of a dropped, unused blocking-job \ + permit, before it exits. Delay shows the native slot is returned \ + by the drop itself, not when this thread ends.", + }, + Site { + name: "reply.close_upgraded", + has_error: false, + description: "The death point's close holds an upgraded deferred slot, before \ + closing it. Delay lets the caller drop its receiver meanwhile: an \ + unread reply must still be destroyed only after every caller is \ + answered (tasks and threads).", + }, + Site { + name: "reply.recv_parked", + has_error: false, + description: "A blocking `ReplyReceiver::recv` is about to wait (under its slot's \ + lock). A handshake: once hit, the receiver is parked and only a \ + notification wakes it.", + }, + Site { + name: "monitor.down_before_dispatch", + has_error: false, + description: "A monitor's `Down` dequeued by its observer (tasks and threads), \ + before the flush check. Delay lets a `demonitor` on another thread \ + return while the `Down` is in dispatch: it must not reach \ + `Handler` (OTP `demonitor(Ref, [flush])`).", + }, Site { name: "child.factory", has_error: true, @@ -141,6 +179,23 @@ pub const SITES: &[Site] = &[ description: "An owned registration is being removed during scope cleanup. Delay \ checks that a replacement only starts after the old name is gone.", }, + Site { + name: "actor.at_death", + has_error: false, + description: "Right after the death point D (the final reason is fixed, admission \ + refuses with `NoProc`), before the open deferred replies are closed \ + (tasks and threads). Delay widens the window in which an external \ + `ReplyTo::reply` races the death: it must not succeed once `NoProc` \ + was observed.", + }, + Site { + name: "actor.exit_before_deliver", + has_error: false, + description: "`ctx.exit(target, reason)` after its sender-lifetime check, before the \ + signal is delivered (tasks and threads). Delay lets the sender reach \ + its death point meanwhile: its death notice must not overtake the \ + signal.", + }, Site { name: "link.before_install", has_error: false, diff --git a/concurrency/src/instrumentation.rs b/concurrency/src/instrumentation.rs index f402c771e..88a5a524b 100644 --- a/concurrency/src/instrumentation.rs +++ b/concurrency/src/instrumentation.rs @@ -1,7 +1,7 @@ //! Internal instrumentation; disabled builds use zero-sized no-op guards. #[cfg(feature = "metrics")] pub(crate) use crate::metrics::{ - accepted, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, + accepted, hooks, reply, ActorObservation, Mailbox, MessageObservation, RequestObservation, }; #[cfg(not(feature = "metrics"))] @@ -29,6 +29,33 @@ mod disabled { #[inline] pub(crate) fn finish(self, _: &Result) {} } + + /// Deferred-reply and job accounting: no-ops without `metrics`. + pub(crate) mod hooks { + pub(crate) type DeferStamp = (); + #[inline(always)] + pub(crate) fn deferred() -> DeferStamp {} + #[inline(always)] + pub(crate) fn deferred_replied(_: &DeferStamp) {} + #[inline(always)] + pub(crate) fn left_open(_: &'static str, _: bool) {} + #[inline(always)] + pub(crate) fn job_started() {} + #[inline(always)] + pub(crate) fn job_ended() {} + #[inline(always)] + pub(crate) fn job_notified(_: &'static str) {} + #[inline(always)] + pub(crate) fn job_cancelled() {} + #[inline(always)] + pub(crate) fn job_fenced() {} + #[inline(always)] + pub(crate) fn job_rejected() {} + #[inline(always)] + pub(crate) fn job_emitted() {} + #[inline(always)] + pub(crate) fn jobs_unfinished(_: usize) {} + } } #[cfg(not(feature = "metrics"))] pub(crate) use disabled::*; diff --git a/concurrency/src/lib.rs b/concurrency/src/lib.rs index 62f27ee60..69a28fe02 100644 --- a/concurrency/src/lib.rs +++ b/concurrency/src/lib.rs @@ -41,7 +41,7 @@ //! - No return type โ€” fire-and-forget send //! //! **Actors** โ€” `#[actor]` on an impl block generates `impl Actor` and `Handler` -//! impls from annotated methods (`#[request_handler]`, `#[send_handler]`, `#[handler]`). +//! impls from annotated methods (`#[request_handler]`, `#[send_handler]`, `#[deferred_handler]`). //! Lifecycle hooks use `#[started]` and `#[stopped]`. //! //! **Type-Erased References** โ€” Each protocol generates an `XRef` type alias @@ -90,12 +90,20 @@ pub mod failpoints; pub mod child_handle; pub mod error; -#[cfg(feature = "async")] +// Test-only: public (and unstable) with the `lifecycle-trace` feature. +#[cfg(feature = "lifecycle-trace")] pub mod lifecycle_trace; +#[cfg(not(feature = "lifecycle-trace"))] +#[allow(dead_code)] +pub(crate) mod lifecycle_trace; pub mod link; pub mod message; pub mod monitor; pub mod registry; +// Reached through the crate root, `tasks` and `threads`; the module path +// is not part of the API. +#[doc(hidden)] +pub mod reply; pub mod response; #[cfg(feature = "async")] pub mod supervision; @@ -106,11 +114,21 @@ pub mod tasks; pub mod threads; pub use child_handle::{ActorId, ChildHandle}; -pub use error::{ActorError, ExitReason}; +pub use error::{ActorError, ExitReason, NoProc}; pub use link::Exit; pub use monitor::{Down, MonitorRef}; -pub use response::Response; -pub use spawned_macros::{actor, protocol}; +pub use reply::{ + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyReceiver, ReplyTo, +}; +pub use response::{Response, DEFAULT_REQUEST_TIMEOUT}; +pub use spawned_macros::{actor, protocol, send_handler}; +/// The instant type of reply and job deadlines (`Deadline::At`, +/// `JobOptions::deadline`, `ReplyTo::deadline`, `JobCtx::emit_at`): +/// `std::time::Instant`, or the simulated clock's instant under +/// `--cfg spawned_sim`. Re-exported so callers need not depend on +/// `spawned-rt`. +pub use spawned_rt::time::Instant; /// Seeded async test attribute: `#[spawned_concurrency::sim_test]`. /// @@ -134,6 +152,63 @@ mod native_link_regression_tests; mod actor_cleanup; mod mailbox; +/// Hooks for `spawned-test` and macro-generated code. Not part of the public +/// API: no semver promise, may change in any release. +#[doc(hidden)] +pub mod __private { + use crate::reply::{Deadline, PendingReply, ReplyReceiver}; + use crate::ActorError; + + /// Seals `Receiver` (tasks and threads): implemented by + /// `ActorRef`, `Context` and spawned-test's probe only. + pub trait Sealed {} + + /// A `PendingReply` not tied to an actor turn, and its receiver + /// (`spawned_test::reply_pair`). + pub fn reply_pair( + deadline: Deadline, + ) -> (PendingReply<'static, R>, ReplyReceiver) { + crate::reply::reply_pair(deadline) + } + + /// The `PendingReply` of a sent message (`spawned_test::detached_reply`). + pub fn detached_reply() -> PendingReply<'static, R> { + PendingReply::detached() + } + + /// Answer `ActorStopped` (a test double standing in for an absent actor). + pub fn discharge_stopped(reply: PendingReply<'_, R>) { + reply.discharge_stopped() + } + + /// Run `f` in one of the actor's turns (`spawned_test::tasks::in_actor`). + #[cfg(feature = "async")] + pub fn in_turn( + actor: &crate::tasks::ActorRef, + f: F, + ) -> Result, ActorError> + where + A: crate::tasks::Actor, + R: Send + 'static, + F: FnOnce(&mut A, &crate::tasks::Context) -> R + Send + 'static, + { + actor.in_turn(f) + } + + /// Run `f` in one of the actor's turns (`spawned_test::threads::in_actor`). + pub fn in_turn_threads( + actor: &crate::threads::ActorRef, + f: F, + ) -> Result, ActorError> + where + A: crate::threads::Actor, + R: Send + 'static, + F: FnOnce(&mut A, &crate::threads::Context) -> R + Send + 'static, + { + actor.in_turn(f) + } +} + mod instrumentation; #[cfg(feature = "metrics")] pub mod metrics; diff --git a/concurrency/src/lifecycle_trace.rs b/concurrency/src/lifecycle_trace.rs index 418d5223e..7ad245852 100644 --- a/concurrency/src/lifecycle_trace.rs +++ b/concurrency/src/lifecycle_trace.rs @@ -1,9 +1,13 @@ -//! Supervisor lifecycle trace recording for TLA+ trace validation. +//! Lifecycle trace recording for TLA+ trace validation. //! //! With the `lifecycle-trace` feature, `run_supervisor`/`run_child` record one //! event per action of `verification/tla/Supervision.tla`, tagged with the -//! supervisor's trace id. `verification/tla/trace_check.py` then checks that -//! each recorded trace is a behavior of the model. +//! supervisor's trace id; the reply and job cores record `DeferredReplies.tla` +//! and `DeferredJobs.tla` actions; and the actor runtime records +//! `Lifecycle.tla` actions (`LifeEvent`, see `TraceLifecycle.tla`) for the +//! actors spawned while `record_lifecycle` is on. +//! `verification/tla/trace_check.py` then checks that each recorded trace is +//! a behavior of its model. //! //! The model's actions are atomic; the code's are not. Every event is //! recorded inside a process-wide *section*, and code that reads the @@ -107,6 +111,147 @@ impl TreeEvent { } } +/// One pipe-to-self job action of `verification/tla/DeferredJobs.tla`, +/// recorded per owner instance. `job` is a process-wide job number +/// (normalized by the checker); `x` is the action's parameter: the message +/// kind (`emit`, `done`, `deadline`) and fence verdict for `Dispatch`, +/// whether the terminal message was enqueued for `Finish` (`notified`, +/// `quiet`, or `held` when scheduled emits keep the result) and `Release`, +/// and whether a delayed emit was enqueued for `Fire` (`emit`, `dropped`; +/// `release` when it was the last and took a held result). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JobEvent { + pub action: &'static str, + pub job: u64, + pub x: &'static str, + pub keyed: bool, + pub deadline: bool, + pub fenced: bool, + pub release: bool, +} + +impl JobEvent { + pub(crate) fn new(action: &'static str, job: u64) -> Self { + Self { + action, + job, + x: "", + keyed: false, + deadline: false, + fenced: false, + release: false, + } + } + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","j":{},"x":"{}","keyed":{},"dl":{},"fenced":{},"rel":{}}}"#, + self.action, self.job, self.x, self.keyed, self.deadline, self.fenced, self.release + ) + } +} + +/// One reply-slot action of `verification/tla/DeferredReplies.tla`, recorded +/// per owner instance: `Send`, `Dispatch`, `Discharge` (with the terminal +/// kind in `x`: `ok`, `timeout`, `dropped`, `stopped`, `overloaded`, +/// `abandoned`), `Exit` and `Death`. `req` identifies the slot (normalized by +/// the checker at each `Send`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReplyEvent { + pub action: &'static str, + pub req: u64, + pub x: &'static str, +} + +impl ReplyEvent { + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","r":{},"x":"{}"}}"#, + self.action, self.req, self.x + ) + } +} + +/// One base-lifecycle action of `verification/tla/Lifecycle.tla`, recorded +/// for the actors spawned while `record_lifecycle` is on. `t` is the actor +/// the action belongs to (the target of a signal or message, the caller of +/// link/monitor, the receiver of a notice), `s` the other actor or the +/// sender (`ActorId` numbers, normalized by the checker; 0 for outside code), +/// `x` the reason, flag, kind or outcome, `m` the monitor, `o` the notice +/// origin (`death`, `late`) or `sup` for a supervisor's signal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LifeEvent { + pub action: &'static str, + pub t: u64, + pub s: u64, + pub x: &'static str, + pub m: u64, + pub k: &'static str, + pub o: &'static str, +} + +impl LifeEvent { + #[allow(dead_code)] // recording is compiled out without the feature + pub(crate) fn new(action: &'static str, t: u64) -> Self { + Self { + action, + t, + s: 0, + x: "", + m: 0, + k: "", + o: "", + } + } + #[allow(dead_code)] + pub(crate) fn s(mut self, s: u64) -> Self { + self.s = s; + self + } + #[allow(dead_code)] + pub(crate) fn x(mut self, x: &'static str) -> Self { + self.x = x; + self + } + #[allow(dead_code)] + pub(crate) fn m(mut self, m: u64) -> Self { + self.m = m; + self + } + #[allow(dead_code)] + pub(crate) fn k(mut self, k: &'static str) -> Self { + self.k = k; + self + } + #[allow(dead_code)] + pub(crate) fn o(mut self, o: &'static str) -> Self { + self.o = o; + self + } + /// JSON object understood by `verification/tla/trace_check.py`. + pub fn to_json(&self) -> String { + format!( + r#"{{"a":"{}","t":{},"s":{},"x":"{}","m":{},"k":"{}","o":"{}"}}"#, + self.action, self.t, self.s, self.x, self.m, self.k, self.o + ) + } +} + +/// Model name of an exit reason (`Lifecycle.tla` `Reasons`). +#[allow(dead_code)] // recording is compiled out without the feature +pub(crate) fn life_reason(reason: &crate::ExitReason) -> &'static str { + use crate::ExitReason; + match reason { + ExitReason::Normal => "Normal", + ExitReason::Shutdown => "Shutdown", + ExitReason::Panic(_) => "Panic", + ExitReason::Kill => "Kill", + ExitReason::Killed => "Killed", + ExitReason::NoProc => "NoProc", + } +} + pub(crate) fn flag(value: bool) -> &'static str { if value { "true" @@ -124,6 +269,9 @@ pub(crate) fn reason(reason: Option<&crate::ExitReason>) -> &'static str { Some(ExitReason::Shutdown) => "Shutdown", Some(ExitReason::Panic(_)) => "Panic", Some(ExitReason::Kill) => "Kill", + // Supervision.tla has one class per restart behaviour: Killed and + // NoProc are abnormal, as Kill. + Some(_) => "Kill", } } @@ -137,7 +285,7 @@ mod imp { cell::Cell, collections::BTreeMap, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, Ordering}, Mutex, MutexGuard, }, }; @@ -155,7 +303,293 @@ mod imp { static TREES: Mutex> = Mutex::new(Vec::new()); static SECTION: Mutex<()> = Mutex::new(()); static TRACES: Mutex>> = Mutex::new(BTreeMap::new()); + static JOBS: Mutex>> = Mutex::new(BTreeMap::new()); + /// Owner instances whose reply-slot and job events are recorded. + static WATCHED: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + + /// Record the reply-slot and job events of owner instance `owner` (an + /// `ActorId`'s number) until [`take_replies`] or [`take_jobs`]. Opt-in, so + /// other tests built with the feature neither accumulate events nor + /// allocate inside the trace section. + pub fn watch_deferred(owner: u64) { + lock(&WATCHED).insert(owner); + } + + fn watched(owner: u64) -> bool { + lock(&WATCHED).contains(&owner) + } + + /// A job number for [`super::JobEvent::job`]. + pub(crate) fn next_job() -> u64 { + static NEXT: AtomicU64 = AtomicU64::new(1); + NEXT.fetch_add(1, Ordering::Relaxed) + } + + /// Record a job action of owner instance `owner` (inside the section). + pub(crate) fn job(owner: u64, event: impl FnOnce() -> super::JobEvent) { + atomic(|| { + if watched(owner) { + lock(&JOBS).entry(owner).or_default().push(event()); + } + }); + } + + static REPLIES: Mutex>> = Mutex::new(BTreeMap::new()); + static DEAD: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + + /// Record a reply-slot action of owner instance `owner`. + pub(crate) fn reply(owner: u64, action: &'static str, req: u64, x: &'static str) { + atomic(|| { + if !watched(owner) { + return; + } + // The owner's death closes everything still open in one model + // step (`Death`); the closes it performs are not separate events. + if action == "Discharge" && x == "stopped" && lock(&DEAD).contains(&owner) { + return; + } + if action == "Death" && !lock(&DEAD).insert(owner) { + return; + } + // A cleanup deadline after the death point (a slow destructor + // before publication) is not a step of the reply registry, which + // is sealed: the model's `CleanupDeadline` precedes death. + if action == "Stuck" && lock(&DEAD).contains(&owner) { + return; + } + lock(&REPLIES) + .entry(owner) + .or_default() + .push(super::ReplyEvent { action, req, x }); + }); + } + + /// Remove and return the reply events recorded for an owner instance. + pub fn take_replies(owner: u64) -> Vec { + lock(&WATCHED).remove(&owner); + lock(&DEAD).remove(&owner); + lock(&REPLIES).remove(&owner).unwrap_or_default() + } + + // Linked jobs and their downstream requests (DeferredReplies.tla + // `JobCallB`, `BDispatch`, `BReply`): while a `pipe_reply*` job's work + // runs, the thread carries its link (owner instance, request slot); a + // request it makes to another actor B is recorded on the owner's trace, + // and B's decisions on that request are recorded there too. thread_local! { + static LINK: Cell> = const { Cell::new(None) }; + } + /// B's slots requested from a linked job: slot -> (owner, request, deferred). + static DOWNSTREAM: Mutex> = Mutex::new(BTreeMap::new()); + + /// Run `f` (a linked job's work, or one poll of it) with its link set. + pub(crate) fn in_link(link: Option<(u64, u64)>, f: impl FnOnce() -> R) -> R { + struct Restore(Option<(u64, u64)>); + impl Drop for Restore { + fn drop(&mut self) { + LINK.with(|l| l.set(self.0)); + } + } + let _restore = Restore(LINK.with(|l| l.replace(link))); + f() + } + + /// A request slot `slot` was created at instance `owner`: if the current + /// thread runs a linked job of a watched instance, it is that request's + /// downstream call (`CallB`). + pub(crate) fn downstream_sent(owner: u64, slot: u64) { + let Some((a, request)) = LINK.with(Cell::get) else { + return; + }; + if a == owner { + return; + } + atomic(|| { + if watched(a) { + lock(&DOWNSTREAM).insert(slot, (a, request, false)); + crate::lifecycle_trace::reply(a, "CallB", request, ""); + } + }); + } + + /// B deferred a downstream request (`BDispatch` to `stateB`). + pub(crate) fn downstream_deferred(slot: u64) { + atomic(|| { + let found = lock(&DOWNSTREAM).get_mut(&slot).map(|entry| { + entry.2 = true; + (entry.0, entry.1) + }); + if let Some((a, request)) = found { + crate::lifecycle_trace::reply(a, "BDispatch", request, "stateB"); + } + }); + } + + /// A downstream request's slot left OPEN with `kind` (inside the slot's + /// trace section): B's answer in its turn (`BDispatch` to `done`) or a + /// later one (`BReply`). The caller's side (abandoned, timeout) is not + /// B's step: the model folds it into the job's end (`EndDown`). + pub(crate) fn downstream_closed(slot: u64, kind: &'static str) { + let Some((a, request, deferred)) = lock(&DOWNSTREAM).remove(&slot) else { + return; + }; + if matches!(kind, "abandoned" | "timeout") { + return; + } + if deferred { + crate::lifecycle_trace::reply(a, "BReply", request, kind); + } else { + crate::lifecycle_trace::reply(a, "BDispatch", request, "done"); + } + } + + /// Remove and return the job events recorded for an owner instance. + pub fn take_jobs(owner: u64) -> Vec { + lock(&WATCHED).remove(&owner); + lock(&JOBS).remove(&owner).unwrap_or_default() + } + + // Base lifecycle (`Lifecycle.tla`): one process-wide recording at a time. + static LIFE_ON: AtomicBool = AtomicBool::new(false); + static MEMBERS: Mutex> = + Mutex::new(std::collections::BTreeSet::new()); + static LIFE: Mutex> = Mutex::new(Vec::new()); + + /// Start recording the base-lifecycle events of every actor spawned from + /// now on until [`take_lifecycle`]. Tests that record must not run + /// concurrently with each other. + pub fn record_lifecycle() { + atomic(|| { + lock(&MEMBERS).clear(); + lock(&LIFE).clear(); + LIFE_ON.store(true, Ordering::Release); + }); + } + + /// Stop recording and return the events in their recorded order (a + /// linearization: each event is recorded in the critical section of the + /// state change it describes). + pub fn take_lifecycle() -> Vec { + atomic(|| { + LIFE_ON.store(false, Ordering::Release); + lock(&MEMBERS).clear(); + std::mem::take(&mut *lock(&LIFE)) + }) + } + + /// Whether actor `id` is recorded. + pub(crate) fn life_traced(id: u64) -> bool { + LIFE_ON.load(Ordering::Acquire) && lock(&MEMBERS).contains(&id) + } + + /// Run `f` as one atomic step of the recorded linearization if actor + /// `id` is recorded (and as is otherwise). Every lifecycle section is + /// keyed by the actor whose locks it takes first, and nested sections use + /// the same key, so the section is always the outermost lock. + pub(crate) fn life_section(id: u64, f: impl FnOnce() -> R) -> R { + if life_traced(id) { + atomic(f) + } else { + f() + } + } + + /// Record an event of actor `t` (inside its section). + pub(crate) fn life_emit(t: u64, event: impl FnOnce() -> super::LifeEvent) { + if life_traced(t) { + lock(&LIFE).push(event()); + } + } + + /// A new actor instance (after its `start_linked` link, before + /// `started()`): recorded from now on. + pub(crate) fn life_spawn(id: u64, parent: u64) { + if !LIFE_ON.load(Ordering::Acquire) { + return; + } + atomic(|| { + lock(&MEMBERS).insert(id); + lock(&LIFE).push(super::LifeEvent::new("Spawn", id).s(parent)); + }); + } + + /// The model's mailbox item kind of a message of type `M`. + pub(crate) fn life_kind() -> &'static str { + if std::any::TypeId::of::() == std::any::TypeId::of::() { + "down" + } else { + "req" + } + } + + /// A send or request to `owner` was admitted or refused (`NoProc`). A + /// `Down` is recorded by its monitor as a notice delivery instead. `from` + /// is the recorded actor that sent it ([`life_sender`]); an admitted + /// send of one is the model's actor-to-actor `Send`, anything else a + /// request from `Ext`. + pub(crate) fn life_send(owner: u64, admitted: bool, from: u64) { + if life_kind::() == "down" { + return; + } + life_emit(owner, || { + super::LifeEvent::new("Send", owner) + .s(if admitted { from } else { 0 }) + .x(if admitted { "queued" } else { "noproc" }) + }); + } + + /// The recorded actor whose turn (or hook) is sending a plain message + /// (not a request, not a `Down`) to another actor `owner`; 0 otherwise. + pub(crate) fn life_sender(owner: u64, is_send: bool) -> u64 { + if !is_send || life_kind::() == "down" { + return 0; + } + let current = CURRENT.with(Cell::get); + if current != 0 && current != owner && life_traced(current) { + current + } else { + 0 + } + } + + /// Run `f` as part of actor `id`'s turn or hook (threads mode). + pub(crate) fn with_actor(id: u64, f: impl FnOnce() -> R) -> R { + struct Restore(u64); + impl Drop for Restore { + fn drop(&mut self) { + CURRENT.with(|c| c.set(self.0)); + } + } + let _restore = Restore(CURRENT.with(|c| c.replace(id))); + f() + } + + type Turn<'a> = std::pin::Pin + Send + 'a>>; + + /// Poll a turn or hook future as part of actor `id` (tasks mode). + pub(crate) fn in_actor(id: u64, turn: Turn<'_>) -> Turn<'_> { + Box::pin(InActor { id, turn }) + } + struct InActor<'a> { + id: u64, + turn: Turn<'a>, + } + impl std::future::Future for InActor<'_> { + type Output = (); + fn poll( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<()> { + let id = self.id; + with_actor(id, || self.turn.as_mut().poll(cx)) + } + } + + thread_local! { + /// The recorded actor whose turn this thread is running. + static CURRENT: Cell = const { Cell::new(0) }; static DEPTH: Cell = const { Cell::new(0) }; static SUPPRESS_CLOSE: Cell = const { Cell::new(false) }; } @@ -247,6 +681,14 @@ mod imp { lock(&TRACES).remove(&id).unwrap_or_default() } + /// Whether a supervisor's trace already holds an event named `action` + /// (lets a test wait for an ordering point without taking the trace). + pub fn recorded(id: u64, action: &str) -> bool { + lock(&TRACES) + .get(&id) + .is_some_and(|events| events.iter().any(|e| e.action == action)) + } + /// What a dumped trace needs besides its events: the constants /// `trace_check.py` instantiates the model with. struct Meta { @@ -429,7 +871,50 @@ mod imp { pub(crate) fn failing(f: impl FnOnce() -> R) -> R { f() } + #[allow(dead_code)] // recording is compiled out without the feature + #[inline(always)] + pub(crate) fn reply(_: u64, _: &'static str, _: u64, _: &'static str) {} + #[allow(dead_code)] // recording is compiled out without the feature + #[inline(always)] + pub(crate) fn job(_: u64, _: impl FnOnce() -> super::JobEvent) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn in_link(_: Option<(u64, u64)>, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_section(_: u64, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_emit(_: u64, _: impl FnOnce() -> super::LifeEvent) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_spawn(_: u64, _: u64) {} + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn life_send(_: u64, _: bool, _: u64) {} + #[allow(dead_code, clippy::extra_unused_type_parameters)] + #[inline(always)] + pub(crate) fn life_sender(_: u64, _: bool) -> u64 { + 0 + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn with_actor(_: u64, f: impl FnOnce() -> R) -> R { + f() + } + #[allow(dead_code)] + #[inline(always)] + pub(crate) fn in_actor(_: u64, turn: T) -> T { + turn + } } #[cfg(feature = "lifecycle-trace")] -pub use imp::{take, take_tree}; +pub use imp::{ + record_lifecycle, recorded, take, take_jobs, take_lifecycle, take_replies, take_tree, + watch_deferred, +}; diff --git a/concurrency/src/link.rs b/concurrency/src/link.rs index 48cf54a8b..978c70e64 100644 --- a/concurrency/src/link.rs +++ b/concurrency/src/link.rs @@ -1,8 +1,9 @@ use crate::child_handle::ActorId; use crate::error::{ActorError, ExitReason}; +use crate::lifecycle_trace as trace; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; // --------------------------------------------------------------------------- // Exit message @@ -16,6 +17,7 @@ use std::sync::{Arc, Mutex, Weak}; /// /// [`Context::trap_exit(true)`]: crate::threads::Context::trap_exit #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub struct Exit { /// The actor that died. pub from: ActorId, @@ -23,6 +25,14 @@ pub struct Exit { pub reason: ExitReason, } +impl Exit { + /// An exit notice from `from` (for test doubles and comparisons; the + /// framework builds the ones it delivers). + pub fn new(from: ActorId, reason: ExitReason) -> Self { + Self { from, reason } + } +} + // --------------------------------------------------------------------------- // Internal types // --------------------------------------------------------------------------- @@ -31,158 +41,350 @@ pub struct Exit { /// mailbox. Captures the peer's typed sender at link time. pub(crate) type SendExitFn = Arc Result<(), ActorError> + Send + Sync>; -/// Per-actor flag controlling how exit signals from linked actors are handled. -/// `false` (default): the receiver is cancelled. `true`: the receiver gets an -/// `Exit` message via `Actor::exit_received`. +/// Queues a signal held during `started()`; `true` for a parent signal. +pub(crate) type SendHeldFn = Arc Result<(), ActorError> + Send + Sync>; + +/// Per-actor flag controlling how exit signals are handled. `false` +/// (default): a non-`Normal` signal ends the receiver. `true`: the receiver +/// gets an `Exit` message via `Actor::exit_received` (only a kill is +/// untrappable). pub(crate) type TrapExitFlag = Arc; -/// Per-actor slot holding the exit reason of a linked actor whose death -/// triggered cancellation. When a non-trapping actor is cancelled by a link -/// signal, this slot is set so the actor's own exit reason propagates -/// transitively through further links. -pub(crate) type LinkedExitReason = Arc>>; +/// The exit request of one actor instance (design otp-exit-signals.md, +/// "Phases of an exit"): the requested reason, kill provenance, whether +/// `stopped()` is skipped, and the death point D after which nothing changes. +#[derive(Debug, Default)] +pub(crate) struct ExitRequest { + reason: Option, + /// A kill signal (or the supervisor's escalation) arrived before D. + killed: bool, + /// The exit was begun by a kill, an untrapped signal or link notice, or + /// `stopped()` must otherwise not start (decision 2). + skip_hook: bool, + hook_begun: bool, + /// Past the death point D: the reason is final. + dead: bool, + /// Phase 0: `started()` is running; non-kill signals are held. + starting: bool, + /// Reasons of the signals held during `started()`, in arrival order. + held: Vec, + /// Held signal envelopes become `Exit`s (the actor trapped when + /// `started()` returned); otherwise they are inert. + held_trapped: bool, + /// A parent signal is among the held ones. + held_parent: bool, + /// A parent exit is queued behind earlier messages of a trapping actor + /// (E3): when dispatched it ends the actor whatever its reason. + parent_exit_queued: bool, +} -/// Create a new empty linked-exit-reason slot. -pub(crate) fn new_linked_exit_reason() -> LinkedExitReason { - Arc::new(Mutex::new(None)) +impl ExitRequest { + /// The requested reason (final after D). + pub(crate) fn reason(&self) -> Option<&ExitReason> { + self.reason.as_ref() + } + pub(crate) fn is_dead(&self) -> bool { + self.dead + } + /// `Context::exit_pending`: exit has begun (a reason is requested, which + /// includes a kill and the dispatch of a parent exit) or a parent exit + /// is queued for a trapping actor. Held phase-0 signals count once they + /// are resolved when `started()` returns. + pub(crate) fn exit_pending(&self) -> bool { + self.reason.is_some() || self.parent_exit_queued + } + /// What began an exit that skips `stopped()`, for its debug log. + pub(crate) fn skip_cause(&self) -> &'static str { + if self.killed { + "kill" + } else { + "exit_signal" + } + } + /// `stopped()` has begun (or was skipped by the run loop). + pub(crate) fn hook_begun(&self) -> bool { + self.hook_begun + } + /// A self-exit or an administrative stop: the first requested reason + /// wins. Returns whether the run loop must be woken. + pub(crate) fn request(&mut self, reason: ExitReason) -> bool { + if self.dead { + return false; + } + if self.reason.is_none() { + self.reason = Some(reason); + } + true + } + /// An untrapped non-`Normal` exit signal or link notice: begins exit + /// without `stopped()`, or replaces the reason of an actor already + /// exiting (decision 8), unless a kill was received. + pub(crate) fn untrapped(&mut self, reason: ExitReason) -> bool { + if self.dead || self.killed { + return false; + } + if self.reason.is_none() && !self.hook_begun { + self.skip_hook = true; + } + self.reason = Some(reason); + true + } + /// An untrappable kill: `Killed` supersedes any reason until D and + /// skips a `stopped()` that has not begun. + pub(crate) fn kill(&mut self) -> bool { + if self.dead { + return false; + } + self.reason = Some(ExitReason::Killed); + self.killed = true; + if !self.hook_begun { + self.skip_hook = true; + } + true + } + /// A panic in `started()`, a handler or `stopped()`: begins exit, or + /// replaces the requested reason, unless a kill was received. + pub(crate) fn panic(&mut self, reason: ExitReason) { + if !self.dead && !self.killed { + self.reason = Some(reason); + } + } + /// `stopped()` is about to start: returns whether it runs. + pub(crate) fn begin_hook(&mut self) -> bool { + self.hook_begun = true; + !self.skip_hook + } + /// The death point D: the requested reason becomes final. + pub(crate) fn die(&mut self) -> ExitReason { + self.dead = true; + self.reason.get_or_insert(ExitReason::Normal).clone() + } + /// The runtime dropped the actor before D (G1): `Killed` unless a + /// non-`Normal` reason was already requested; then D. + pub(crate) fn runtime_drop(&mut self) -> ExitReason { + if !self.dead { + self.reason = match self.reason.take() { + None | Some(ExitReason::Normal) => Some(ExitReason::Killed), + Some(reason) => Some(reason), + }; + } + self.die() + } + pub(crate) fn begin_starting(&mut self) { + self.starting = true; + } + /// `started()` returned: resolve the held signals with the trap flag read + /// now. A non-trapping actor begins exit with the first non-`Normal` + /// one; a trapping actor dispatches each held envelope as an `Exit` in + /// its own place in the queue. Returns whether the run loop must stop. + pub(crate) fn end_starting(&mut self, trapping: bool) -> bool { + self.starting = false; + let held = std::mem::take(&mut self.held); + self.held_trapped = trapping; + if trapping && std::mem::take(&mut self.held_parent) { + // The held parent envelope is dispatched as a parent exit. + self.parent_exit_queued = true; + } + if !trapping { + if let Some(reason) = held.into_iter().find(|r| *r != ExitReason::Normal) { + return self.untrapped(reason); + } + } + false + } + pub(crate) fn held_trapped(&self) -> bool { + self.held_trapped + } } -/// An entry in an actor's link table, representing one linked peer. -pub(crate) struct LinkEntry { - /// The peer's unique actor ID. - pub peer_id: ActorId, - /// Delivers an exit signal to the peer. - pub signal: ExitSignalFn, - /// Reference to the peer's link table, so we can remove ourselves on death. - pub peer_links: Weak>, +/// Per-actor exit request, shared by every handle of the instance (and by its +/// managed scope). +pub(crate) type LinkedExitReason = Arc>; + +/// Create a new empty exit request. +pub(crate) fn new_linked_exit_reason() -> LinkedExitReason { + Arc::new(Mutex::new(ExitRequest::default())) } -/// Type-erased function that signals exit to a peer. Decides whether to -/// cancel the peer or send an `Exit` message based on the peer's `trap_exit` -/// flag and the exit reason. -pub(crate) type ExitSignalFn = Arc; - -/// An actor's link table โ€” a list of linked peers. -#[derive(Default)] -pub(crate) struct LinkState { - entries: Vec, - // Published under the same mutex as registration, before edges are drained. - // Completion channels can be published later without losing a racing link. - exited: Option, +pub(crate) fn lock_request(request: &LinkedExitReason) -> std::sync::MutexGuard<'_, ExitRequest> { + request.lock().unwrap_or_else(|p| p.into_inner()) } -pub(crate) type LinkTable = Arc>; -/// Create a new empty link table. -pub(crate) fn new_link_table() -> LinkTable { - Arc::new(Mutex::new(LinkState::default())) +/// Everything needed to deliver an exit signal to one actor instance without +/// knowing its type. +pub(crate) struct SignalTarget { + pub(crate) id: ActorId, + /// The actor's `start_linked` parent: its signals are parent exits. + pub(crate) parent: Option, + pub(crate) trap: TrapExitFlag, + pub(crate) request: LinkedExitReason, + /// Wakes the run loop (cancels its token) without recording a reason. + pub(crate) wake: Arc, + /// Queues an `Exit` for `exit_received`. + pub(crate) send_exit: SendExitFn, + /// Queues a parent exit: when dispatched, exit begins with its reason and + /// `stopped()` runs; `exit_received` never sees it (E3). + pub(crate) send_parent_exit: SendExitFn, + /// Queues a signal held during `started()` in its place in the mailbox + /// (the flag marks a parent signal). + pub(crate) send_held: SendHeldFn, } -/// Create a new trap_exit flag (default: false). -pub(crate) fn new_trap_exit_flag() -> TrapExitFlag { - Arc::new(AtomicBool::new(false)) +/// Where an exit signal comes from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Origin { + /// `ctx.exit` / `ChildHandle::exit` / `kill` (a `Kill` is untrappable). + Direct, + /// A death notice of a link, or a late `NoProc` (a `Kill` is trappable). + Link, } -/// Build an `ExitSignalFn` that signals a peer based on its trap_exit flag -/// and a typed-erased send-exit function. -pub(crate) fn make_signal( - peer_trap_exit: TrapExitFlag, - peer_cancel: Arc, - peer_send_exit: SendExitFn, - peer_linked_reason: LinkedExitReason, -) -> ExitSignalFn { - Arc::new(move |sender_id, reason| { - // A propagated notice is trappable, including Kill. Only a direct - // stop_with_reason(Kill) bypasses the target's trap_exit flag. - // Normal exit signals are silently dropped unless the peer is trapping - let trapping = peer_trap_exit.load(Ordering::Acquire); - if matches!(reason, ExitReason::Normal) && !trapping { - return; - } - if trapping { - // Send Exit message to the peer's mailbox - let exit = Exit { - from: sender_id, - reason, - }; - let _ = peer_send_exit(exit); // mailbox may be closed if peer just died - } else { - // Peer is not trapping: record the linked reason then cancel - let mut slot = peer_linked_reason.lock().unwrap_or_else(|p| p.into_inner()); - if slot.is_none() { - *slot = Some(reason); - } - drop(slot); - peer_cancel(); +/// Deliver an exit signal (OTP `exit_signal/2` semantics, design E1-E3). +/// `from_parent` marks the internal entry point of supervisors and systems. +pub(crate) fn deliver_signal( + target: &SignalTarget, + from: ActorId, + reason: ExitReason, + origin: Origin, + from_parent: bool, +) { + trace::life_section(target.id.raw(), || { + if origin == Origin::Direct { + trace::life_emit(target.id.raw(), || { + trace::LifeEvent::new("Signal", target.id.raw()) + .s(from.raw()) + .x(trace::life_reason(&reason)) + .o(if from_parent { "sup" } else { "" }) + }); } + apply_signal(target, from, reason, origin, from_parent) }) } -/// Mutate both ends in one transaction. Callbacks and removed entries are -/// handled after releasing both locks. Pointer order is stable while Arcs live. -fn with_pair( - own: &LinkTable, - peer: &LinkTable, - f: impl FnOnce(&mut LinkState, &mut LinkState) -> R, -) -> R { - if Arc::as_ptr(own) < Arc::as_ptr(peer) { - let mut a = own.lock().unwrap_or_else(|p| p.into_inner()); - let mut b = peer.lock().unwrap_or_else(|p| p.into_inner()); - f(&mut a, &mut b) - } else { - let mut b = peer.lock().unwrap_or_else(|p| p.into_inner()); - let mut a = own.lock().unwrap_or_else(|p| p.into_inner()); - f(&mut a, &mut b) +fn apply_signal( + target: &SignalTarget, + from: ActorId, + reason: ExitReason, + origin: Origin, + from_parent: bool, +) { + if origin == Origin::Direct && reason == ExitReason::Kill { + if lock_request(&target.request).kill() { + (target.wake)(); + } + return; + } + let from_parent = from_parent || target.parent == Some(from); + let mut request = lock_request(&target.request); + if request.dead { + return; + } + let exit = Exit { from, reason }; + if request.starting { + // Phase 0 is not in the lifecycle trace mapping (TraceLifecycle.tla). + trace::life_emit(target.id.raw(), || { + trace::LifeEvent::new("Held", target.id.raw()).s(from.raw()) + }); + // Held in arrival order, and as an envelope in its mailbox place. + request.held.push(exit.reason.clone()); + request.held_parent |= from_parent; + let _ = (target.send_held)(exit, from_parent); + return; + } + // The trap flag is read when the signal arrives. + if target.trap.load(Ordering::Acquire) { + if from_parent { + // Set before the envelope is queued, under the request lock, so + // `exit_pending` reports it to every handler that runs before it. + request.parent_exit_queued = true; + } + drop(request); + let send = if from_parent { + &target.send_parent_exit + } else { + &target.send_exit + }; + let _ = send(exit); // admission may be closed if the peer just died + return; + } + if exit.reason == ExitReason::Normal { + return; } + if request.untrapped(exit.reason) { + drop(request); + (target.wake)(); + } +} + +mod sync { + pub(crate) use std::sync::{Arc, Mutex, MutexGuard, Weak}; +} +mod table; +pub(crate) use table::{new_link_table, LinkNotices, LinkOutcome, LinkTable}; + +/// Create a new trap_exit flag (default: false). +pub(crate) fn new_trap_exit_flag() -> TrapExitFlag { + Arc::new(AtomicBool::new(false)) } -/// Register before actor execution starts, or atomically observe a closed peer. -/// Late operations through an already-closed own context are ignored. +static NEXT_EDGE: AtomicU64 = AtomicU64::new(1); + +/// Link `own` and `peer` (OTP `link/1`). An existing record (live or +/// pending) makes it a no-op. Past the peer's D, `try_only` returns +/// `NoProc` without a record; otherwise a pending late record is created and +/// its `NoProc` notice is delivered through the receiver-side step, so a +/// concurrent `unlink` still fences it. pub(crate) fn register_link( - own_id: ActorId, - own: &LinkTable, - own_signal: ExitSignalFn, - peer_id: ActorId, - peer: &LinkTable, - peer_signal: ExitSignalFn, -) { - if own_id == peer_id || Arc::ptr_eq(own, peer) { - return; + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + try_only: bool, +) -> LinkOutcome { + if own.id == peer.id || Arc::ptr_eq(own_links, peer_links) { + return LinkOutcome::Unchanged; + } + trace::life_section(own.id.raw(), || { + link_in_section(own, own_links, peer, peer_links, try_only) + }) +} + +fn link_in_section( + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + try_only: bool, +) -> LinkOutcome { + // Sender lifetime (design Terms, semantics X2): `link` works only while + // the caller runs. Once its exit began (or past its D, checked again + // under the table lock below) the call is inert: no record, no notice, + // no NoProc. A dead process cannot call link in OTP. + if lock_request(&own.request).reason().is_some() { + return LinkOutcome::Unchanged; } fail_point!("link.before_install"); - let closed = with_pair(own, peer, |a, b| { - if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer_id) { - return None; - } - if let Some(reason) = &b.exited { - return Some(reason.clone()); - } - a.entries.push(LinkEntry { - peer_id, - signal: peer_signal.clone(), - peer_links: Arc::downgrade(peer), - }); - b.entries.push(LinkEntry { - peer_id: own_id, - signal: own_signal.clone(), - peer_links: Arc::downgrade(own), + let gen = NEXT_EDGE.fetch_add(1, Ordering::Relaxed); + let outcome = table::install(own, own_links, peer, peer_links, gen, try_only); + if outcome != LinkOutcome::Unchanged { + trace::life_emit(own.id.raw(), || { + trace::LifeEvent::new("Link", own.id.raw()) + .s(peer.id.raw()) + .x(match (&outcome, try_only) { + (LinkOutcome::Linked, _) => "linked", + (_, false) => "noproc", + (_, true) => "try", + }) }); - None - }); - if let Some(reason) = closed { - deliver(&own_signal, peer_id, reason); } + if outcome == LinkOutcome::NoProc && !try_only { + deliver_notice(own_links, own, peer.id, gen, ExitReason::NoProc); + } + outcome } -fn remove(table: &mut LinkState, id: ActorId) -> Option { - table - .entries - .iter() - .position(|e| e.peer_id == id) - .map(|i| table.entries.swap_remove(i)) -} - -/// Unlink is atomic with registration. An exit already taken for delivery may -/// still arrive; unlink does not retract a signal already in flight. +/// Unlink removes both records, live or pending: once it returns, no notice +/// of that edge takes effect (E6). An `Exit` already queued stays. pub(crate) fn unregister_link( own_id: ActorId, own: &LinkTable, @@ -192,80 +394,124 @@ pub(crate) fn unregister_link( if own_id == peer_id || Arc::ptr_eq(own, peer) { return; } - let removed = with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))); + let removed = trace::life_section(own_id.raw(), || { + let removed = table::remove_pair(own_id, own, peer_id, peer); + if removed.0.is_some() || removed.1.is_some() { + trace::life_emit(own_id.raw(), || { + trace::LifeEvent::new("Unlink", own_id.raw()).s(peer_id.raw()) + }); + } + removed + }); drop(removed); } -fn deliver(signal: &ExitSignalFn, id: ActorId, reason: ExitReason) { - // One failing internal cancellation callback must not prevent other peers - // from being notified, or panic again during actor-future destruction. - if let Err(payload) = catch_unwind(AssertUnwindSafe(|| signal(id, reason))) { - crate::mailbox::dispose_panic(payload); +/// Receiver-side delivery: under the receiver's table lock, apply the notice +/// only if the record with that peer and generation is still present, and +/// remove it. `unlink` removes records under the same lock (the fence). +fn deliver_notice( + receiver_links: &LinkTable, + receiver: &SignalTarget, + from: ActorId, + gen: u64, + reason: ExitReason, +) { + trace::life_section(receiver.id.raw(), || { + deliver_notice_in_section(receiver_links, receiver, from, gen, reason) + }) +} + +fn deliver_notice_in_section( + receiver_links: &LinkTable, + receiver: &SignalTarget, + from: ActorId, + gen: u64, + reason: ExitReason, +) { + let record = table::take_record(receiver_links, from, gen, |record| { + trace::life_emit(receiver.id.raw(), || { + trace::LifeEvent::new("Deliver", receiver.id.raw()) + .s(from.raw()) + .x(trace::life_reason(&reason)) + .k("link") + .o(if record.late { "late" } else { "death" }) + }); + // One failing internal callback must not prevent other peers from + // being notified, or panic again during actor-future destruction. if let Err(payload) = catch_unwind(AssertUnwindSafe(|| { - tracing::error!("Panic while delivering a linked exit"); + deliver_signal(receiver, from, reason, Origin::Link, false) })) { crate::mailbox::dispose_panic(payload); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic while delivering a linked exit"); + }); } - } + }); + drop(record); } -/// Close the registration state before draining. Terminal closure is idempotent. -pub(crate) fn propagate_exit(own_id: ActorId, own: &LinkTable, reason: &ExitReason) { - let entries = { - let mut state = own.lock().unwrap_or_else(|p| p.into_inner()); - if state.exited.is_some() { - return; - } - state.exited = Some(reason.clone()); - std::mem::take(&mut state.entries) - }; - for entry in entries { - let removed = entry.peer_links.upgrade().and_then(|peer| { - let mut table = peer.lock().unwrap_or_else(|p| p.into_inner()); - remove(&mut table, own_id) - }); - drop(removed); - deliver(&entry.signal, own_id, reason.clone()); +impl LinkNotices { + /// Deliver every notice (publication). Each is applied only if the + /// receiver still holds the record (E6 fence). + pub(crate) fn deliver(self) { + self.deliver_with(deliver_notice); } } -/// Created before constructing the actor future/thread closure, so shutdown -/// before its first poll also closes links. Weak reverse edges avoid Arc cycles. -pub(crate) struct LinkCleanup { - id: ActorId, - links: LinkTable, +/// Close the exit table at D: set the final reason and take one notice per +/// link. Takes only this actor's own lock. Terminal closure is idempotent. +#[cfg(test)] +pub(crate) fn close_links(own_id: ActorId, own: &LinkTable, reason: &ExitReason) -> LinkNotices { + close_links_at_death(own_id, own, reason, |_| {}) } -impl LinkCleanup { - pub(crate) fn new(id: ActorId, links: LinkTable) -> Self { - Self { id, links } - } - pub(crate) fn complete(&self, reason: &ExitReason) { - propagate_exit(self.id, &self.links, reason); - } + +/// The exit-table part of the death point D. `at_death` runs inside the same +/// critical section (it must not run user code): the reply registry seals +/// the final reason there, so a registration that finds the table closed +/// (`NoProc`) and a send that finds the registry sealed (`NoProc`) agree. +pub(crate) fn close_links_at_death( + own_id: ActorId, + own: &LinkTable, + reason: &ExitReason, + at_death: impl FnOnce(&ExitReason), +) -> LinkNotices { + table::close(own_id, own, reason, at_death) } -impl Drop for LinkCleanup { - fn drop(&mut self) { - self.complete(&ExitReason::Kill); - } + +/// Close and deliver at once (tests and paths with nothing in between). +#[cfg(test)] +pub(crate) fn propagate_exit(own_id: ActorId, own: &LinkTable, reason: &ExitReason) { + close_links(own_id, own, reason).deliver(); } -/// A local stop commits its cause before cancelling. A linked failure recorded -/// first wins; a late linked failure cannot relabel an already-requested stop. -pub(crate) fn normal_stop(reason: &LinkedExitReason, cancel: impl FnOnce()) { - stop_with_reason(reason, ExitReason::Normal, cancel); +/// A self-exit or administrative stop commits its cause before waking. The +/// first requested reason wins over later self-exits and stops; a later +/// untrapped signal or kill can still replace it until D. +pub(crate) fn normal_stop(id: ActorId, reason: &LinkedExitReason, cancel: impl FnOnce()) { + stop_with_reason(id, reason, ExitReason::Normal, cancel); } +/// A self-exit or administrative stop of actor `id` (recorded as `Stop`). pub(crate) fn stop_with_reason( + id: ActorId, reason: &LinkedExitReason, cause: ExitReason, cancel: impl FnOnce(), ) { - let mut slot = reason.lock().unwrap_or_else(|p| p.into_inner()); - if slot.is_none() { - *slot = Some(cause); + trace::life_section(id.raw(), || { + trace::life_emit(id.raw(), || { + trace::LifeEvent::new("Stop", id.raw()).x(trace::life_reason(&cause)) + }); + request_stop(reason, cause, cancel); + }); +} + +/// As [`stop_with_reason`], unrecorded: a parent exit being dispatched (part +/// of the model's `Dispatch`) or a managed scope's stop. +pub(crate) fn request_stop(reason: &LinkedExitReason, cause: ExitReason, cancel: impl FnOnce()) { + if lock_request(reason).request(cause) { + cancel(); } - drop(slot); - cancel(); } #[cfg(test)] @@ -332,11 +578,15 @@ mod tests { } let drops = Arc::new(AtomicUsize::new(0)); let callback_drops = drops.clone(); - let signal: ExitSignalFn = - Arc::new(move |_, _| std::panic::panic_any(Payload(callback_drops.clone()))); + let mut receiver = target(&Arc::new(AtomicUsize::new(0))); + receiver.wake = Arc::new(move || std::panic::panic_any(Payload(callback_drops.clone()))); + let receiver = Arc::new(receiver); + let sender = Arc::new(target(&Arc::new(AtomicUsize::new(0)))); + let (a, b) = (new_link_table(), new_link_table()); + register_link(&receiver, &a, &sender, &b, false); let result = catch_unwind(AssertUnwindSafe(|| { tracing::subscriber::with_default(PanicSubscriber(drops.clone()), || { - deliver(&signal, ActorId::next(), ExitReason::Kill); + propagate_exit(sender.id, &b, &ExitReason::Kill); }); })); assert!(result.is_ok(), "delivery unwound through its caller"); @@ -367,94 +617,276 @@ mod tests { fn trap_exit_defaults_false() { assert!(!new_trap_exit_flag().load(Ordering::Acquire)); } - fn signal(counter: &Arc) -> ExitSignalFn { - let c = counter.clone(); - Arc::new(move |_, _| { - c.fetch_add(1, Ordering::SeqCst); - }) + /// A receiver whose wake and Exit deliveries are counted. + fn target(counter: &Arc) -> SignalTarget { + let (c1, c2, c3) = (counter.clone(), counter.clone(), counter.clone()); + SignalTarget { + id: ActorId::next(), + parent: None, + trap: new_trap_exit_flag(), + request: new_linked_exit_reason(), + wake: Arc::new(move || { + c1.fetch_add(1, Ordering::SeqCst); + }), + send_exit: Arc::new(move |_| { + c2.fetch_add(1, Ordering::SeqCst); + Ok(()) + }), + send_parent_exit: Arc::new(move |_| { + c3.fetch_add(1, Ordering::SeqCst); + Ok(()) + }), + send_held: Arc::new(|_, _| Ok(())), + } + } + fn pair( + seen: &Arc, + ) -> (Arc, LinkTable, Arc, LinkTable) { + ( + Arc::new(target(seen)), + new_link_table(), + Arc::new(target(seen)), + new_link_table(), + ) } #[test] fn closure_is_terminal_and_delivers_each_edge_once() { - let a = new_link_table(); - let b = new_link_table(); - let aid = ActorId::next(); - let bid = ActorId::next(); let seen = Arc::new(AtomicUsize::new(0)); - register_link(aid, &a, signal(&seen), bid, &b, signal(&seen)); - propagate_exit(aid, &a, &ExitReason::Kill); - propagate_exit(aid, &a, &ExitReason::Kill); + let (a, at, b, bt) = pair(&seen); + assert_eq!(register_link(&a, &at, &b, &bt, false), LinkOutcome::Linked); + propagate_exit(a.id, &at, &ExitReason::Panic("x".into())); + propagate_exit(a.id, &at, &ExitReason::Panic("x".into())); assert_eq!(seen.load(Ordering::SeqCst), 1); - assert!(b.lock().unwrap().entries.is_empty()); - register_link(bid, &b, signal(&seen), aid, &a, signal(&seen)); + assert!(bt.lock().unwrap().entries.is_empty()); + // b links to the dead a: a late NoProc notice, delivered and removed. + // b is exiting (the notice began its exit): its link is inert + // (model: Link requires a running caller). + assert_eq!( + register_link(&b, &bt, &a, &at, false), + LinkOutcome::Unchanged + ); + assert_eq!(seen.load(Ordering::SeqCst), 1); + // A running c linking to the dead a gets the late NoProc notice. + let c = Arc::new(target(&seen)); + let ct = new_link_table(); + assert_eq!(register_link(&c, &ct, &a, &at, false), LinkOutcome::NoProc); assert_eq!(seen.load(Ordering::SeqCst), 2); - assert!(a.lock().unwrap().entries.is_empty() && b.lock().unwrap().entries.is_empty()); + assert_eq!(lock_request(&c.request).reason(), Some(&ExitReason::NoProc)); + assert!(ct.lock().unwrap().entries.is_empty()); + assert!(at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty()); + } + #[test] + fn try_link_to_a_dead_actor_keeps_no_record() { + let seen = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&seen); + propagate_exit(b.id, &bt, &ExitReason::Normal); + assert_eq!(register_link(&a, &at, &b, &bt, true), LinkOutcome::NoProc); + assert_eq!(seen.load(Ordering::SeqCst), 0); + assert!(at.lock().unwrap().entries.is_empty()); + assert_eq!(lock_request(&a.request).reason(), None); + } + #[test] + fn unlink_fences_a_notice_taken_at_death() { + let seen = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&seen); + register_link(&a, &at, &b, &bt, false); + let notices = close_links(a.id, &at, &ExitReason::Panic("x".into())); + unregister_link(b.id, &bt, a.id, &at); + notices.deliver(); + assert_eq!( + seen.load(Ordering::SeqCst), + 0, + "unlink returned before delivery" + ); + // A relink while the notice is pending is a no-op, as an existing record. + let (c, ct, d, dt) = pair(&seen); + register_link(&c, &ct, &d, &dt, false); + let notices = close_links(c.id, &ct, &ExitReason::Panic("y".into())); + assert_eq!( + register_link(&d, &dt, &c, &ct, false), + LinkOutcome::Unchanged + ); + notices.deliver(); + assert_eq!(seen.load(Ordering::SeqCst), 1); + } + #[test] + fn signal_rules_follow_exit_signal_2() { + let seen = Arc::new(AtomicUsize::new(0)); + let t = target(&seen); + let from = ActorId::next(); + // Normal is ignored by a non-trapping target. + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), None); + // Any other reason begins exit without stopped(). + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert_eq!( + lock_request(&t.request).reason(), + Some(&ExitReason::Shutdown) + ); + assert!(!lock_request(&t.request).begin_hook()); + // Trapping: queued as Exit, nothing committed. + let t = target(&seen); + t.trap.store(true, Ordering::Release); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), None); + // A Kill signal is untrappable and ends Killed; a Kill link notice is trappable. + deliver_signal(&t, from, ExitReason::Kill, Origin::Link, false); + assert_eq!(lock_request(&t.request).reason(), None); + deliver_signal(&t, from, ExitReason::Kill, Origin::Direct, false); + assert_eq!(lock_request(&t.request).reason(), Some(&ExitReason::Killed)); + } + #[test] + fn exit_pending_counts_begun_exits_and_queued_parent_exits() { + let seen = Arc::new(AtomicUsize::new(0)); + let from = ActorId::next(); + let pending = |t: &SignalTarget| lock_request(&t.request).exit_pending(); + // Trapping: a peer's Exit is not pending; a parent's signal is. + let t = target(&seen); + t.trap.store(true, Ordering::Release); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Link, false); + assert!(!pending(&t)); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(pending(&t)); + assert_eq!(lock_request(&t.request).reason(), None); + // Non-trapping: an untrapped signal begins exit; Normal is ignored. + let t = target(&seen); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(!pending(&t)); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + assert!(pending(&t)); + // Phase 0: held signals count when started() returns. + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + assert!(!pending(&t)); + lock_request(&t.request).end_starting(true); + assert!(pending(&t), "a held parent exit of a trapping actor"); + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + deliver_signal(&t, from, ExitReason::Shutdown, Origin::Direct, false); + lock_request(&t.request).end_starting(false); + assert!(pending(&t), "a held untrapped signal begins exit"); + let t = target(&seen); + lock_request(&t.request).begin_starting(); + deliver_signal(&t, from, ExitReason::Normal, Origin::Direct, true); + lock_request(&t.request).end_starting(false); + assert!( + !pending(&t), + "a held Normal parent signal is ignored untrapped" + ); + } + #[test] + fn reason_precedence_until_death() { + crate::test_wait::bounded( + concat!(module_path!(), "::reason_precedence_until_death"), + move || { + // A late non-kill signal replaces a requested reason (decision 8); + // a kill supersedes it and nothing replaces Killed. + let mut r = ExitRequest::default(); + assert!(r.request(ExitReason::Normal)); + assert!(r.untrapped(ExitReason::Shutdown)); + assert_eq!(r.reason(), Some(&ExitReason::Shutdown)); + assert!( + r.begin_hook(), + "a late signal does not skip a self-exit's stopped()" + ); + r.panic(ExitReason::Panic("stopped".into())); + assert_eq!(r.reason(), Some(&ExitReason::Panic("stopped".into()))); + assert!(r.kill()); + r.panic(ExitReason::Panic("after kill".into())); + assert!(!r.untrapped(ExitReason::Shutdown)); + assert!(r.request(ExitReason::Normal)); + assert_eq!(r.die(), ExitReason::Killed); + assert!(!r.kill() && !r.untrapped(ExitReason::Shutdown)); + // Runtime drop: Killed unless a non-Normal reason was requested. + let mut r = ExitRequest::default(); + r.request(ExitReason::Normal); + assert_eq!(r.runtime_drop(), ExitReason::Killed); + let mut r = ExitRequest::default(); + r.request(ExitReason::Shutdown); + assert_eq!(r.runtime_drop(), ExitReason::Shutdown); + }, + ); + } + #[test] + fn link_from_an_exiting_or_dead_caller_is_inert() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_from_an_exiting_or_dead_caller_is_inert" + ), + move || { + let seen = Arc::new(AtomicUsize::new(0)); + // Exit begun: no record on either side, even to a live peer. + let (a, at, b, bt) = pair(&seen); + lock_request(&a.request).request(ExitReason::Normal); + assert_eq!( + register_link(&a, &at, &b, &bt, false), + LinkOutcome::Unchanged + ); + assert!( + at.lock().unwrap().entries.is_empty() && bt.lock().unwrap().entries.is_empty() + ); + // Past D, even to a dead peer: no NoProc notice, no try_link error. + let (c, ct, d, dt) = pair(&seen); + lock_request(&c.request).die(); + propagate_exit(c.id, &ct, &ExitReason::Normal); + propagate_exit(d.id, &dt, &ExitReason::Normal); + assert_eq!( + register_link(&c, &ct, &d, &dt, false), + LinkOutcome::Unchanged + ); + assert_eq!( + register_link(&c, &ct, &d, &dt, true), + LinkOutcome::Unchanged + ); + assert_eq!(seen.load(Ordering::SeqCst), 0); + }, + ); } #[test] fn self_link_is_noop() { - let a = new_link_table(); - let id = ActorId::next(); let seen = Arc::new(AtomicUsize::new(0)); - register_link(id, &a, signal(&seen), id, &a, signal(&seen)); - assert!(a.lock().unwrap().entries.is_empty()); - assert_eq!(Arc::strong_count(&a), 1); + let a = Arc::new(target(&seen)); + let at = new_link_table(); + register_link(&a, &at, &a, &at, false); + assert!(at.lock().unwrap().entries.is_empty()); + assert_eq!(Arc::strong_count(&at), 1); } #[test] fn weak_edges_do_not_keep_link_tables_alive() { - let a = new_link_table(); - let b = new_link_table(); - let weak_a = Arc::downgrade(&a); - let weak_b = Arc::downgrade(&b); let c = Arc::new(AtomicUsize::new(0)); - register_link( - ActorId::next(), - &a, - signal(&c), - ActorId::next(), - &b, - signal(&c), - ); - drop(a); - drop(b); + let (a, at, b, bt) = pair(&c); + let weak_a = Arc::downgrade(&at); + let weak_b = Arc::downgrade(&bt); + register_link(&a, &at, &b, &bt, false); + drop(at); + drop(bt); assert!(weak_a.upgrade().is_none() && weak_b.upgrade().is_none()); } #[test] fn concurrent_link_unlink_preserves_symmetry() { - let a = new_link_table(); - let b = new_link_table(); - let aid = ActorId::next(); - let bid = ActorId::next(); let c = Arc::new(AtomicUsize::new(0)); + let (a, at, b, bt) = pair(&c); std::thread::scope(|scope| { scope.spawn(|| { for _ in 0..2000 { - register_link(aid, &a, signal(&c), bid, &b, signal(&c)); - unregister_link(aid, &a, bid, &b); + register_link(&a, &at, &b, &bt, false); + unregister_link(a.id, &at, b.id, &bt); } }); scope.spawn(|| { for _ in 0..2000 { - register_link(bid, &b, signal(&c), aid, &a, signal(&c)); - unregister_link(bid, &b, aid, &a); + register_link(&b, &bt, &a, &at, false); + unregister_link(b.id, &bt, a.id, &at); } }); }); assert_eq!( - a.lock().unwrap().entries.len(), - b.lock().unwrap().entries.len() - ); - assert!(a.lock().unwrap().entries.len() <= 1); - } - #[test] - fn normal_stop_cause_is_not_replaced_by_late_failure() { - let reason = new_linked_exit_reason(); - normal_stop(&reason, || {}); - let signal = make_signal( - new_trap_exit_flag(), - Arc::new(|| {}), - Arc::new(|_| Ok(())), - reason.clone(), + at.lock().unwrap().entries.len(), + bt.lock().unwrap().entries.len() ); - signal(ActorId::next(), ExitReason::Panic("late".into())); - assert_eq!(*reason.lock().unwrap(), Some(ExitReason::Normal)); + assert!(at.lock().unwrap().entries.len() <= 1); } } diff --git a/concurrency/src/link/table.rs b/concurrency/src/link/table.rs new file mode 100644 index 000000000..4f6c7b99c --- /dev/null +++ b/concurrency/src/link/table.rs @@ -0,0 +1,226 @@ +//! The link table protocol (docs/design/otp-exit-signals.md E5, E6): edge +//! generations, both records installed in one two-lock transaction, +//! receiver-side fenced delivery and the close at the death point D that +//! takes the notices. +//! +//! Shared with the Loom model (`tests/link_fence_loom.rs`), which supplies +//! `ActorId`, `ExitReason`, `SignalTarget` and the synchronization types +//! through `super`. The protocol never runs user code under a table lock +//! except through the `apply` callback of [`take_record`], which production +//! uses to apply the signal to the receiver (it takes only the receiver's +//! exit-request lock, never a table lock). + +use super::sync::{Arc, Mutex, MutexGuard, Weak}; +use super::{ActorId, ExitReason, SignalTarget}; + +/// An entry in an actor's link table, representing one linked peer. +pub(crate) struct LinkEntry { + /// The peer's unique actor ID. + pub peer_id: ActorId, + /// Edge generation: both records of one link share it; a late record has + /// its own. Delivery applies only to a record with this generation. + pub gen: u64, + /// The peer, to deliver this actor's death notice to it. + pub peer: Arc, + /// Reference to the peer's link table, for receiver-side delivery. + pub peer_links: Weak>, + /// A pending late `NoProc` notice (the peer was past D at `link`). + pub late: bool, +} + +/// An actor's exit table: its link records and its death point. +#[derive(Default)] +pub(crate) struct LinkState { + pub(super) entries: Vec, + // Set at the death point D under the same mutex as registration. + pub(super) exited: Option, +} +pub(crate) type LinkTable = Arc>; + +impl LinkState { + /// Past the death point D. + pub(crate) fn is_dead(&self) -> bool { + self.exited.is_some() + } +} + +/// Create a new empty link table. +pub(crate) fn new_link_table() -> LinkTable { + Arc::new(Mutex::new(LinkState::default())) +} + +pub(super) fn lock(table: &Mutex) -> MutexGuard<'_, LinkState> { + table.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// Mutate both ends in one transaction. Callbacks and removed entries are +/// handled after releasing both locks. Pointer order is stable while Arcs live. +fn with_pair( + own: &LinkTable, + peer: &LinkTable, + f: impl FnOnce(&mut LinkState, &mut LinkState) -> R, +) -> R { + if Arc::as_ptr(own) < Arc::as_ptr(peer) { + let mut a = lock(own); + let mut b = lock(peer); + f(&mut a, &mut b) + } else { + let mut b = lock(peer); + let mut a = lock(own); + f(&mut a, &mut b) + } +} + +/// The outcome of a link attempt. +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum LinkOutcome { + Linked, + /// Nothing to do: self link, an existing record, or the caller's exit + /// began (inert, as `monitor`). + Unchanged, + /// The peer was past D. + NoProc, +} + +/// Install the records of edge `gen` between `own` and `peer`. An existing +/// record (live or pending) or a closed own table makes it a no-op. Past the +/// peer's D, `try_only` installs nothing; otherwise a pending late record is +/// installed on `own`, whose `NoProc` notice the caller delivers through +/// [`take_record`] (so a concurrent `unlink` still fences it). +pub(crate) fn install( + own: &Arc, + own_links: &LinkTable, + peer: &Arc, + peer_links: &LinkTable, + gen: u64, + try_only: bool, +) -> LinkOutcome { + with_pair(own_links, peer_links, |a, b| { + if a.exited.is_some() || a.entries.iter().any(|e| e.peer_id == peer.id) { + return LinkOutcome::Unchanged; + } + if b.exited.is_some() { + if !try_only { + a.entries.push(LinkEntry { + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: true, + }); + } + return LinkOutcome::NoProc; + } + a.entries.push(LinkEntry { + peer_id: peer.id, + gen, + peer: peer.clone(), + peer_links: Arc::downgrade(peer_links), + late: false, + }); + b.entries.push(LinkEntry { + peer_id: own.id, + gen, + peer: own.clone(), + peer_links: Arc::downgrade(own_links), + late: false, + }); + LinkOutcome::Linked + }) +} + +fn remove(table: &mut LinkState, id: ActorId) -> Option { + table + .entries + .iter() + .position(|e| e.peer_id == id) + .map(|i| table.entries.swap_remove(i)) +} + +/// Remove both records of the edge, live or pending, in one transaction: +/// once this returns no notice of that edge takes effect (E6). The caller +/// drops the removed records after every lock is released. +pub(crate) fn remove_pair( + own_id: ActorId, + own: &LinkTable, + peer_id: ActorId, + peer: &LinkTable, +) -> (Option, Option) { + with_pair(own, peer, |a, b| (remove(a, peer_id), remove(b, own_id))) +} + +/// Receiver-side delivery: under the receiver's table lock, apply the notice +/// only if the record with that peer and generation is still present, and +/// remove it. `unlink` and the receiver's own close at D remove records +/// under the same lock (the fence). Returns the removed record, which the +/// caller drops after the lock is released. +pub(crate) fn take_record( + receiver_links: &LinkTable, + from: ActorId, + gen: u64, + apply: impl FnOnce(&LinkEntry), +) -> Option { + let mut table = lock(receiver_links); + let i = table + .entries + .iter() + .position(|e| e.peer_id == from && e.gen == gen)?; + let record = table.entries.swap_remove(i); + apply(&record); + drop(table); + Some(record) +} + +/// Death notices taken at D, delivered at publication. +#[derive(Default)] +pub(crate) struct LinkNotices { + from: Option, + reason: Option, + entries: Vec, +} + +impl LinkNotices { + /// Deliver every notice (publication) through `notice(receiver table, + /// receiver, from, gen, reason)`, which applies it only if the receiver + /// still holds the record (E6 fence). Late records carry no notice of + /// this actor's death: their `NoProc` was this actor's own. + pub(crate) fn deliver_with( + self, + mut notice: impl FnMut(&LinkTable, &SignalTarget, ActorId, u64, ExitReason), + ) { + let (Some(from), Some(reason)) = (self.from, self.reason) else { + return; + }; + for entry in self.entries { + if entry.late { + continue; + } + if let Some(peer_links) = entry.peer_links.upgrade() { + notice(&peer_links, &entry.peer, from, entry.gen, reason.clone()); + } + } + } +} + +/// The exit-table part of the death point D: set the final reason and take +/// one notice per link, taking only this actor's own lock. `at_death` runs +/// inside the same critical section (it must not run user code). Terminal +/// closure is idempotent: a second close takes nothing. +pub(crate) fn close( + own_id: ActorId, + own: &LinkTable, + reason: &ExitReason, + at_death: impl FnOnce(&ExitReason), +) -> LinkNotices { + let mut state = lock(own); + if state.exited.is_some() { + return LinkNotices::default(); + } + at_death(reason); + state.exited = Some(reason.clone()); + LinkNotices { + from: Some(own_id), + reason: Some(reason.clone()), + entries: std::mem::take(&mut state.entries), + } +} diff --git a/concurrency/src/link_regression_tests.rs b/concurrency/src/link_regression_tests.rs index a30d849a1..914267093 100644 --- a/concurrency/src/link_regression_tests.rs +++ b/concurrency/src/link_regression_tests.rs @@ -24,54 +24,63 @@ fn runtime() -> tokio::runtime::Runtime { } #[test] -fn linking_in_post_drain_pre_completion_window_delivers_exit() { - let rt = runtime(); - let events = Arc::new(Mutex::new(Vec::new())); - let observer = { - let _entered = rt.enter(); - Observer(events.clone()).start() - }; - observer.context().trap_exit(true); - let peer_id = ActorId::next(); - let peer_links = link::new_link_table(); - let (_completion_tx, completion_rx) = spawned_rt::tasks::watch::channel(None); - // This is the actual state between target propagation and publication. - let reason = ExitReason::Panic("closing before completion".into()); - link::propagate_exit(peer_id, &peer_links, &reason); - let peer = ChildHandle::from_tasks( - peer_id, - spawned_rt::tasks::CancellationToken::new(), - completion_rx, - link::new_trap_exit_flag(), - peer_links, - link::new_linked_exit_reason(), - Arc::new(|_| Ok(())), - ); - observer.context().link(&peer); - let delivered = rt.block_on(async { - tokio::time::timeout(crate::test_wait::WAIT, async { - loop { - if !events.lock().unwrap().is_empty() { - break; - } - tokio::task::yield_now().await; - } - }) - .await - .is_ok() - }); - observer.child_handle().stop(); - rt.block_on(observer.wait_exit()); - assert!( - delivered, - "exit lost between link-table drain and completion publication" - ); - assert_eq!( - *events.lock().unwrap(), - vec![Exit { - from: peer_id, - reason - }] +fn linking_between_death_point_and_publication_delivers_noproc() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::linking_between_death_point_and_publication_delivers_noproc" + ), + move || { + let rt = runtime(); + let events = Arc::new(Mutex::new(Vec::new())); + let observer = { + let _entered = rt.enter(); + Observer(events.clone()).start() + }; + observer.context().trap_exit(true); + let peer_id = ActorId::next(); + let peer_links = link::new_link_table(); + let (_completion_tx, completion_rx) = spawned_rt::tasks::watch::channel(None); + // This is the actual state between the death point D and publication: + // a registration after D reports NoProc, not the real reason (E4). + let real = ExitReason::Panic("closing before completion".into()); + link::propagate_exit(peer_id, &peer_links, &real); + let reason = ExitReason::NoProc; + let peer = ChildHandle::new( + crate::child_handle::tests::test_target( + peer_id, + spawned_rt::threads::CancellationToken::new(), + ), + crate::child_handle::Completion::Tasks(completion_rx), + peer_links, + ); + observer.context().link(&peer); + let delivered = rt.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, async { + loop { + if !events.lock().unwrap().is_empty() { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .is_ok() + }); + observer.child_handle().stop(); + rt.block_on(observer.wait_exit()); + assert!( + delivered, + "exit lost between link-table drain and completion publication" + ); + assert_eq!( + *events.lock().unwrap(), + vec![Exit { + from: peer_id, + reason + }] + ); + }, ); } @@ -79,38 +88,47 @@ struct Idler; impl Actor for Idler {} #[test] fn never_polled_actor_propagates_runtime_shutdown_to_peer() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; // Those process-owned pools deliberately outlive the caller runtime. - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_actor_propagates_runtime_shutdown_to_peer" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; // Those process-owned pools deliberately outlive the caller runtime. + } + } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = { - let _entered = survivor.enter(); - Idler.start() - }; - peer.context().link(&target.child_handle()); - // The owner runtime never polls its actor. Dropping it must still close links. - drop(owner); - let result = survivor - .block_on(async { tokio::time::timeout(crate::test_wait::WAIT, peer.wait_exit()).await }); - peer.child_handle().stop(); - survivor.block_on(peer.wait_exit()); - assert_eq!( - result, - Ok(ExitReason::Kill), - "aborted actor failed to signal its live peer" + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = { + let _entered = survivor.enter(); + Idler.start() + }; + peer.context().link(&target.child_handle()); + // The owner runtime never polls its actor. Dropping it must still close links. + drop(owner); + let result = survivor.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, peer.wait_exit()).await + }); + peer.child_handle().stop(); + survivor.block_on(peer.wait_exit()); + assert_eq!( + result, + Ok(ExitReason::Killed), + "aborted actor failed to signal its live peer" + ); + }, ); } @@ -119,81 +137,111 @@ impl crate::message::Message for Ping { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn polled_actor_propagates_runtime_shutdown_to_peer() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - owner.block_on(target.request(Ping)).unwrap(); - let peer = { - let _entered = survivor.enter(); - Idler.start() - }; - peer.context().link(&target.child_handle()); - drop(owner); - let result = survivor - .block_on(async { tokio::time::timeout(Duration::from_secs(1), peer.wait_exit()).await }); - peer.child_handle().stop(); - survivor.block_on(peer.wait_exit()); - assert_eq!(result, Ok(ExitReason::Kill)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::polled_actor_propagates_runtime_shutdown_to_peer" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + owner.block_on(target.request(Ping)).unwrap(); + let peer = { + let _entered = survivor.enter(); + Idler.start() + }; + peer.context().link(&target.child_handle()); + drop(owner); + let result = survivor.block_on(async { + tokio::time::timeout(Duration::from_secs(1), peer.wait_exit()).await + }); + peer.child_handle().stop(); + survivor.block_on(peer.wait_exit()); + assert_eq!(result, Ok(ExitReason::Killed)); + }, + ); } struct MonitorObserver(Option>); impl Actor for MonitorObserver {} impl crate::tasks::Handler for MonitorObserver { - async fn handle(&mut self, down: crate::Down, _: &Context) { + async fn handle( + &mut self, + down: crate::Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.0.take().unwrap().send(down).unwrap(); + reply.reply(()); } } #[test] fn never_polled_actor_notifies_monitor_on_runtime_shutdown() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let (tx, rx) = spawned_rt::tasks::oneshot::channel(); - let observer = { - let _entered = survivor.enter(); - MonitorObserver(Some(tx)).start() - }; - { - let _entered = survivor.enter(); - observer.context().monitor(&target.child_handle()); - } - drop(owner); - let result = - survivor.block_on(async { tokio::time::timeout(Duration::from_secs(1), rx).await }); - observer.child_handle().stop(); - survivor.block_on(observer.wait_exit()); - assert_eq!(result.unwrap().unwrap().reason, ExitReason::Kill); + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_actor_notifies_monitor_on_runtime_shutdown" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + let owner = runtime(); + let survivor = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let (tx, rx) = spawned_rt::tasks::oneshot::channel(); + let observer = { + let _entered = survivor.enter(); + MonitorObserver(Some(tx)).start() + }; + { + let _entered = survivor.enter(); + observer.context().monitor(&target.child_handle()); + } + drop(owner); + let result = + survivor.block_on(async { tokio::time::timeout(Duration::from_secs(1), rx).await }); + observer.child_handle().stop(); + survivor.block_on(observer.wait_exit()); + assert_eq!(result.unwrap().unwrap().reason, ExitReason::Killed); + }, + ); } struct Stop; @@ -205,8 +253,15 @@ impl crate::message::Message for Crash { type Result = (); } impl crate::tasks::Handler for Idler { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("linked failure during another actor's stopped hook"); + async fn handle( + &mut self, + _: Crash, + _reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("linked failure during another actor's stopped hook"); + } } } struct Stopping { @@ -220,41 +275,61 @@ impl Actor for Stopping { } } impl crate::tasks::Handler for Stopping { - async fn handle(&mut self, _: Stop, ctx: &Context) { + async fn handle( + &mut self, + _: Stop, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop(); + reply.reply(()); } } +/// Inverted in v0.8 (decision 8, as OTP: a process in `terminate` dies with +/// a later exit signal): the abnormal link notice that arrives while +/// `stopped()` runs replaces the requested `Normal`; the running `stopped()` +/// still finishes, and the linked survivor receives the new reason. #[test] -fn late_link_failure_cannot_relabel_an_already_requested_normal_stop() { - let rt = runtime(); - let (entered_tx, entered_rx) = spawned_rt::tasks::oneshot::channel(); - let (release_tx, release_rx) = spawned_rt::tasks::oneshot::channel(); - let (stopping, failing, survivor) = { - let _entered = rt.enter(); - ( - Stopping { - entered: Some(entered_tx), - release: Some(release_rx), - } - .start(), - Idler.start(), - Idler.start(), - ) - }; - stopping.context().link(&failing.child_handle()); - stopping.context().link(&survivor.child_handle()); - stopping.send(Stop).unwrap(); - rt.block_on(entered_rx).unwrap(); - failing.send(Crash).unwrap(); - assert!(matches!( - rt.block_on(failing.wait_exit()), - ExitReason::Panic(_) - )); - release_tx.send(()).unwrap(); - assert_eq!(rt.block_on(stopping.wait_exit()), ExitReason::Normal); - assert!(survivor.child_handle().is_alive()); - survivor.child_handle().stop(); - rt.block_on(survivor.wait_exit()); +fn late_link_failure_relabels_an_already_requested_normal_stop() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::late_link_failure_relabels_an_already_requested_normal_stop" + ), + move || { + let rt = runtime(); + let (entered_tx, entered_rx) = spawned_rt::tasks::oneshot::channel(); + let (release_tx, release_rx) = spawned_rt::tasks::oneshot::channel(); + let (stopping, failing, survivor) = { + let _entered = rt.enter(); + ( + Stopping { + entered: Some(entered_tx), + release: Some(release_rx), + } + .start(), + Idler.start(), + Idler.start(), + ) + }; + stopping.context().link(&failing.child_handle()); + stopping.context().link(&survivor.child_handle()); + stopping.send(Stop).unwrap(); + rt.block_on(entered_rx).unwrap(); + failing.send(Crash).unwrap(); + assert!(matches!( + rt.block_on(failing.wait_exit()), + ExitReason::Panic(_) + )); + release_tx.send(()).unwrap(); + let relabelled = rt.block_on(stopping.wait_exit()); + assert!( + matches!(&relabelled, ExitReason::Panic(m) if m.contains("linked failure")), + "got {relabelled:?}" + ); + assert_eq!(rt.block_on(survivor.wait_exit()), relabelled); + }, + ); } struct LinkedAtStart { @@ -274,266 +349,325 @@ impl Actor for LinkedAtStart { } #[test] fn start_linked_registers_before_async_started() { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(2) - .enable_all() - .build() - .unwrap(); - let parent = { - let _entered = rt.enter(); - Idler.start() - }; - for _ in 0..32 { - let (tx, rx) = spawned_rt::tasks::oneshot::channel(); - let child = { - let _entered = rt.enter(); - LinkedAtStart { - parent: parent.id(), - seen: Some(tx), + crate::test_wait::bounded( + concat!( + module_path!(), + "::start_linked_registers_before_async_started" + ), + move || { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let parent = { + let _entered = rt.enter(); + Idler.start() + }; + for _ in 0..32 { + let (tx, rx) = spawned_rt::tasks::oneshot::channel(); + let child = { + let _entered = rt.enter(); + LinkedAtStart { + parent: parent.id(), + seen: Some(tx), + } + .start_linked(&parent.context()) + }; + assert!(rt.block_on(rx).unwrap()); + assert_eq!(rt.block_on(child.wait_exit()), ExitReason::Normal); } - .start_linked(&parent.context()) - }; - assert!(rt.block_on(rx).unwrap()); - assert_eq!(rt.block_on(child.wait_exit()), ExitReason::Normal); - } - parent.child_handle().stop(); - rt.block_on(parent.wait_exit()); + parent.child_handle().stop(); + rt.block_on(parent.wait_exit()); + }, + ); } #[test] fn never_polled_monitor_owner_releases_waiter_and_refuses_late_registration() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner_rt = runtime(); - let survivor = runtime(); - let (tx, _rx) = spawned_rt::tasks::oneshot::channel(); - let owner = { - let _entered = owner_rt.enter(); - MonitorObserver(Some(tx)).start() - }; - let target = { - let _entered = survivor.enter(); - Idler.start() - }; - let handle = target.child_handle(); - let probe = handle.lifetime_probe(); - { - let _entered = survivor.enter(); - owner.context().monitor(&handle); - } - drop(handle); - drop(owner_rt); - survivor.block_on(async { - tokio::time::timeout(Duration::from_secs(1), async { - while probe.upgrade().is_some() { - tokio::task::yield_now().await; + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_monitor_owner_releases_waiter_and_refuses_late_registration" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } } - }) - .await - .unwrap(); - }); - let handle = target.child_handle(); - let probe = handle.lifetime_probe(); - { - let _entered = survivor.enter(); - owner.context().monitor(&handle); - } - drop(handle); - assert!( - probe.upgrade().is_none(), - "dead owner retained a new monitor waiter" + let owner_rt = runtime(); + let survivor = runtime(); + let (tx, _rx) = spawned_rt::tasks::oneshot::channel(); + let owner = { + let _entered = owner_rt.enter(); + MonitorObserver(Some(tx)).start() + }; + let target = { + let _entered = survivor.enter(); + Idler.start() + }; + let handle = target.child_handle(); + let probe = handle.lifetime_probe(); + { + let _entered = survivor.enter(); + owner.context().monitor(&handle); + } + drop(handle); + drop(owner_rt); + survivor.block_on(async { + tokio::time::timeout(Duration::from_secs(1), async { + while probe.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + }); + let handle = target.child_handle(); + let probe = handle.lifetime_probe(); + { + let _entered = survivor.enter(); + owner.context().monitor(&handle); + } + drop(handle); + assert!( + probe.upgrade().is_none(), + "dead owner retained a new monitor waiter" + ); + target.child_handle().stop(); + survivor.block_on(target.wait_exit()); + assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Killed)); + }, ); - target.child_handle().stop(); - survivor.block_on(target.wait_exit()); - assert_eq!(owner.child_handle().exit_reason(), Some(ExitReason::Kill)); } #[test] fn linked_runtime_abort_is_trappable() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - let owner = runtime(); - let survivor = runtime(); - let events = Arc::new(Mutex::new(Vec::new())); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = { - let _entered = survivor.enter(); - Observer(events.clone()).start() - }; - peer.context().trap_exit(true); - peer.context().link(&target.child_handle()); - drop(owner); - survivor.block_on(async { - tokio::time::timeout(crate::test_wait::WAIT, async { - while events.lock().unwrap().is_empty() { - tokio::time::sleep(Duration::from_millis(1)).await; + crate::test_wait::bounded( + concat!(module_path!(), "::linked_runtime_abort_is_trappable"), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } } - }) - .await - .unwrap(); - }); - assert!(peer.child_handle().is_alive()); - assert_eq!( - *events.lock().unwrap(), - vec![Exit { - from: target.id(), - reason: ExitReason::Kill - }] + let owner = runtime(); + let survivor = runtime(); + let events = Arc::new(Mutex::new(Vec::new())); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = { + let _entered = survivor.enter(); + Observer(events.clone()).start() + }; + peer.context().trap_exit(true); + peer.context().link(&target.child_handle()); + drop(owner); + survivor.block_on(async { + tokio::time::timeout(crate::test_wait::WAIT, async { + while events.lock().unwrap().is_empty() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + }); + assert!(peer.child_handle().is_alive()); + assert_eq!( + *events.lock().unwrap(), + vec![Exit { + from: target.id(), + reason: ExitReason::Killed + }] + ); + peer.child_handle().stop(); + assert_eq!(survivor.block_on(peer.wait_exit()), ExitReason::Normal); + }, ); - peer.child_handle().stop(); - assert_eq!(survivor.block_on(peer.wait_exit()), ExitReason::Normal); } #[test] fn never_polled_completion_waits_for_actor_payload_destruction() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - struct BlockingDrop { - entered: std::sync::mpsc::Sender<()>, - release: std::sync::mpsc::Receiver<()>, - } - impl Actor for BlockingDrop {} - impl Drop for BlockingDrop { - fn drop(&mut self) { - let _ = self.entered.send(()); - let _ = self.release.recv(); - } - } - let owner = runtime(); - let (entered_tx, entered_rx) = std::sync::mpsc::channel(); - let (release_tx, release_rx) = std::sync::mpsc::channel(); - let actor = { - let _entered = owner.enter(); - BlockingDrop { - entered: entered_tx, - release: release_rx, - } - .start() - }; - let shutdown = std::thread::spawn(move || drop(owner)); - let entered = entered_rx.recv_timeout(Duration::from_secs(2)); - let before = actor.child_handle().exit_reason(); - let _ = release_tx.send(()); - shutdown.join().unwrap(); - entered.unwrap(); - assert_eq!( - before, None, - "completion published while the actor's destructor was still running" + crate::test_wait::bounded( + concat!( + module_path!(), + "::never_polled_completion_waits_for_actor_payload_destruction" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + struct BlockingDrop { + entered: std::sync::mpsc::Sender<()>, + release: std::sync::mpsc::Receiver<()>, + } + impl Actor for BlockingDrop {} + impl Drop for BlockingDrop { + fn drop(&mut self) { + let _ = self.entered.send(()); + let _ = self.release.recv(); + } + } + let owner = runtime(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let actor = { + let _entered = owner.enter(); + BlockingDrop { + entered: entered_tx, + release: release_rx, + } + .start() + }; + let shutdown = std::thread::spawn(move || drop(owner)); + let entered = entered_rx.recv_timeout(Duration::from_secs(2)); + let before = actor.child_handle().exit_reason(); + let _ = release_tx.send(()); + shutdown.join().unwrap(); + entered.unwrap(); + assert_eq!( + before, None, + "completion published while the actor's destructor was still running" + ); + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); + }, ); - assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Kill)); } #[test] fn linked_runtime_abort_cancels_an_idle_threads_actor() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - use crate::threads::{Actor as ThreadActor, ActorStart as ThreadStart}; - struct ThreadPeer; - impl ThreadActor for ThreadPeer {} - impl crate::threads::Handler for ThreadPeer { - fn handle(&mut self, _: Ping, _: &crate::threads::Context) {} - } - let owner = runtime(); - let waiter = runtime(); - let target = { - let _entered = owner.enter(); - Idler.start() - }; - let peer = ThreadPeer.start(); - peer.context().link(&target.child_handle()); - drop(owner); - let result = waiter.block_on(async { - tokio::time::timeout( - Duration::from_secs(2), - peer.child_handle().wait_exit_async(), - ) - .await - }); - peer.child_handle().stop(); - if result.is_err() { - let _ = peer.send(Ping); // Wake cleanup even if the cancellation callback regresses. - } - peer.child_handle().wait_exit_blocking(); - assert_eq!(result, Ok(ExitReason::Kill)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::linked_runtime_abort_cancels_an_idle_threads_actor" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + use crate::threads::{Actor as ThreadActor, ActorStart as ThreadStart}; + struct ThreadPeer; + impl ThreadActor for ThreadPeer {} + impl crate::threads::Handler for ThreadPeer { + fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &crate::threads::Context, + ) { + reply.reply(()); + } + } + let owner = runtime(); + let waiter = runtime(); + let target = { + let _entered = owner.enter(); + Idler.start() + }; + let peer = ThreadPeer.start(); + peer.context().link(&target.child_handle()); + drop(owner); + let result = waiter.block_on(async { + tokio::time::timeout( + Duration::from_secs(2), + peer.child_handle().wait_exit_async(), + ) + .await + }); + peer.child_handle().stop(); + if result.is_err() { + let _ = peer.send(Ping); // Wake cleanup even if the cancellation callback regresses. + } + peer.child_handle().wait_exit_blocking(); + assert_eq!(result, Ok(ExitReason::Killed)); + }, + ); } #[test] fn dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor() { - #[cfg(feature = "backend-experiments")] - { - let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); - if selector == "sharded" - || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) - .is_some() - { - return; - } - } - use std::sync::atomic::{AtomicUsize, Ordering}; - struct Node(Arc); - impl Actor for Node {} - impl crate::tasks::Handler for Node { - async fn handle(&mut self, _: Ping, _: &Context) {} - } - impl Drop for Node { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - let owner = runtime(); - let dropped = Arc::new(AtomicUsize::new(0)); - let actors: Vec<_> = { - let _entered = owner.enter(); - (0..4096).map(|_| Node(dropped.clone()).start()).collect() - }; - for i in 0..actors.len() { - actors[i] - .context() - .link(&actors[(i + 1) % actors.len()].child_handle()); - } - owner.block_on(async { - for actor in &actors { - actor.request(Ping).await.unwrap(); - } - }); - drop(owner); - assert_eq!(dropped.load(Ordering::SeqCst), actors.len()); - assert!(actors - .iter() - .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Kill))); + crate::test_wait::bounded( + concat!( + module_path!(), + "::dropping_runtime_with_large_polled_link_cycle_reclaims_every_actor" + ), + move || { + #[cfg(feature = "backend-experiments")] + { + let selector = std::env::var("SPAWNED_EXPERIMENT_SCHEDULER").unwrap_or_default(); + if selector == "sharded" + || spawned_rt::tasks::experimental::dispatcher::policy_for_scheduler(&selector) + .is_some() + { + return; + } + } + use std::sync::atomic::{AtomicUsize, Ordering}; + struct Node(Arc); + impl Actor for Node {} + impl crate::tasks::Handler for Node { + async fn handle( + &mut self, + _: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } + } + impl Drop for Node { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + let owner = runtime(); + let dropped = Arc::new(AtomicUsize::new(0)); + let actors: Vec<_> = { + let _entered = owner.enter(); + (0..4096).map(|_| Node(dropped.clone()).start()).collect() + }; + for i in 0..actors.len() { + actors[i] + .context() + .link(&actors[(i + 1) % actors.len()].child_handle()); + } + owner.block_on(async { + for actor in &actors { + actor.request(Ping).await.unwrap(); + } + }); + drop(owner); + assert_eq!(dropped.load(Ordering::SeqCst), actors.len()); + assert!(actors + .iter() + .all(|a| a.child_handle().exit_reason() == Some(ExitReason::Killed))); + }, + ); } diff --git a/concurrency/src/mailbox.rs b/concurrency/src/mailbox.rs index 2cace291f..044e7e3a9 100644 --- a/concurrency/src/mailbox.rs +++ b/concurrency/src/mailbox.rs @@ -7,19 +7,61 @@ use std::{ }, }; +/// An owner's sink for the user payloads its framework code drops (refused +/// or undelivered messages, job outcomes, scheduled emits): each drop is +/// contained, its first failure recorded. From the start of the owner's +/// exit until its death point has answered every caller, drops are held +/// instead ([`hold`](Self::hold), [`release`](Self::release)): no user +/// destructor runs before the last answer, on whatever thread drops it. #[derive(Default)] -pub(crate) struct DropFailures(Mutex>); +pub(crate) struct DropFailures { + first: Mutex>, + held: Mutex>>>, +} impl DropFailures { pub(crate) fn first(&self) -> Option { - self.0.lock().unwrap_or_else(|p| p.into_inner()).clone() + self.first.lock().unwrap_or_else(|p| p.into_inner()).clone() + } + /// Record a cleanup failure; the first one is kept. + pub(crate) fn record(&self, message: String) { + self.first + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_or_insert(message); + } + /// The owner's exit began: hold every later drop until `release`. + pub(crate) fn hold(&self) { + self.held + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get_or_insert_with(Vec::new); + } + /// Every caller was answered: drop what was held (contained), and drop + /// at once from now on. Idempotent. + pub(crate) fn release(&self) { + let held = self.held.lock().unwrap_or_else(|p| p.into_inner()).take(); + for value in held.into_iter().flatten() { + self.drop_contained(value); + } + } + pub(crate) fn discard(&self, value: T) { + let mut held = self.held.lock().unwrap_or_else(|p| p.into_inner()); + if let Some(held) = held.as_mut() { + // Moved, not dropped: no user code under the lock. + held.push(Box::new(value)); + return; + } + drop(held); + self.drop_contained(value); } - pub(crate) fn discard(&self, value: T) { + /// Drop at once (contained): the mailbox's own items, destroyed after + /// the death point's discharge. + pub(crate) fn discard_now(&self, value: T) { + self.drop_contained(value); + } + fn drop_contained(&self, value: T) { if let Err(payload) = catch_unwind(AssertUnwindSafe(|| drop(value))) { - let message = crate::error::panic_message(&*payload); - self.0 - .lock() - .unwrap_or_else(|p| p.into_inner()) - .get_or_insert(message); + self.record(crate::error::panic_message(&*payload)); dispose_panic(payload); } } @@ -37,7 +79,8 @@ pub(crate) fn dispose_panic(payload: Box) { #[path = "mailbox_admission.rs"] mod admission; -use admission::SendGate; +/// Also the death point's fences (reply and exit-signal admission). +pub(crate) use admission::{SendGate, SendPermit}; #[derive(Default)] struct Admission { gate: SendGate, @@ -98,17 +141,34 @@ pub(crate) mod tasks { &mut self.inner } } - impl Drop for Receiver { - fn drop(&mut self) { + impl Receiver { + /// Close admission and take everything accepted, without dropping + /// it: the death point answers queued requests before any payload + /// destructor runs ([`discard_all`](Self::discard_all) then). + pub(crate) fn drain(&mut self) -> Vec { self.inner.close(); + let mut items = Vec::new(); // Sends remain synchronous and unbounded. EOF waits only for raw // reservations to publish, never for this runtime thread to run work. // Suppress Tokio's coop deferral while the owning task is destroyed. block_on_drop(spawned_rt::tasks::unconstrained(async { while let Some(value) = self.inner.recv().await { - self.failures.discard(value); + items.push(value); } })); + items + } + /// Destroy drained items; a destructor panic is a cleanup failure. + pub(crate) fn discard_all(&self, items: Vec) { + for item in items { + self.failures.discard_now(item); + } + } + } + impl Drop for Receiver { + fn drop(&mut self) { + let items = self.drain(); + self.discard_all(items); } } pub(crate) fn channel() -> (Sender, Receiver) { @@ -170,14 +230,30 @@ pub(crate) mod threads { &self.inner } } - impl Drop for Receiver { - fn drop(&mut self) { + impl Receiver { + /// Close admission (waiting for in-flight sends) and take everything + /// accepted, without dropping it (see the async receiver). + pub(crate) fn drain(&mut self) -> Vec { self.admission.close(); + let mut items = Vec::new(); while let Ok(value) = self.inner.try_recv() { - self.admission.failures.discard(value); + items.push(value); + } + items + } + /// Destroy drained items; a destructor panic is a cleanup failure. + pub(crate) fn discard_all(&self, items: Vec) { + for item in items { + self.admission.failures.discard_now(item); } } } + impl Drop for Receiver { + fn drop(&mut self) { + let items = self.drain(); + self.discard_all(items); + } + } pub(crate) fn channel() -> (Sender, Receiver) { let (send, recv) = raw::channel(); let admission = Arc::new(Admission::default()); @@ -224,152 +300,186 @@ mod tests { #[test] fn accepted_sends_are_drained_even_when_senders_outlive_receiver() { - struct Item { - dropped: Arc, - accepted: bool, - } - impl Drop for Item { - fn drop(&mut self) { - if self.accepted { - self.dropped.fetch_add(1, Ordering::SeqCst); + crate::test_wait::bounded( + concat!( + module_path!(), + "::accepted_sends_are_drained_even_when_senders_outlive_receiver" + ), + move || { + struct Item { + dropped: Arc, + accepted: bool, } - } - } - macro_rules! check { - ($channel:path) => { - for _ in 0..50 { - let (sender, receiver) = $channel(); - let accepted = Arc::new(AtomicUsize::new(1)); - let dropped = Arc::new(AtomicUsize::new(0)); - assert!(sender - .send(Item { - dropped: dropped.clone(), - accepted: true - }) - .is_ok()); - let gate = Arc::new(std::sync::Barrier::new(3)); - let publishers: Vec<_> = (0..2) - .map(|_| { - let sender = sender.clone(); - let accepted = accepted.clone(); - let dropped = dropped.clone(); - let gate = gate.clone(); - std::thread::spawn(move || { - gate.wait(); - for _ in 0..64 { - match sender.send(Item { - dropped: dropped.clone(), - accepted: true, - }) { - Ok(()) => { - accepted.fetch_add(1, Ordering::SeqCst); - } - Err(mut rejected) => { - rejected.0.accepted = false; - } - } - } - }) - }) - .collect(); - gate.wait(); - drop(receiver); - for publisher in publishers { - publisher.join().unwrap(); + impl Drop for Item { + fn drop(&mut self) { + if self.accepted { + self.dropped.fetch_add(1, Ordering::SeqCst); + } } - assert_eq!( - accepted.load(Ordering::SeqCst), - dropped.load(Ordering::SeqCst) - ); - assert!(sender.failures().first().is_none()); } - }; - } - check!(threads::channel); - #[cfg(feature = "async")] - check!(tasks::channel); + macro_rules! check { + ($channel:path) => { + for _ in 0..50 { + let (sender, receiver) = $channel(); + let accepted = Arc::new(AtomicUsize::new(1)); + let dropped = Arc::new(AtomicUsize::new(0)); + assert!(sender + .send(Item { + dropped: dropped.clone(), + accepted: true + }) + .is_ok()); + let gate = Arc::new(std::sync::Barrier::new(3)); + let publishers: Vec<_> = (0..2) + .map(|_| { + let sender = sender.clone(); + let accepted = accepted.clone(); + let dropped = dropped.clone(); + let gate = gate.clone(); + std::thread::spawn(move || { + gate.wait(); + for _ in 0..64 { + match sender.send(Item { + dropped: dropped.clone(), + accepted: true, + }) { + Ok(()) => { + accepted.fetch_add(1, Ordering::SeqCst); + } + Err(mut rejected) => { + rejected.0.accepted = false; + } + } + } + }) + }) + .collect(); + gate.wait(); + drop(receiver); + for publisher in publishers { + publisher.join().unwrap(); + } + assert_eq!( + accepted.load(Ordering::SeqCst), + dropped.load(Ordering::SeqCst) + ); + assert!(sender.failures().first().is_none()); + } + }; + } + check!(threads::channel); + #[cfg(feature = "async")] + check!(tasks::channel); + }, + ); } #[test] fn close_waits_for_publication_and_rejects_late_senders() { - let (sender, receiver) = threads::channel(); - let held = sender.admission.enter().unwrap(); - let (done, completion) = mpsc::channel(); - let owner = std::thread::spawn(move || { - drop(receiver); - done.send(()).unwrap(); - }); - let deadline = std::time::Instant::now() + Duration::from_secs(5); - while !sender.admission.gate.is_closed() { - assert!(std::time::Instant::now() < deadline, "close did not start"); - std::thread::yield_now(); - } - assert!(sender.send(1).is_err()); - assert!(completion.try_recv().is_err()); - // Model a producer paused between reservation and raw publication. - sender.inner.send(2).unwrap(); - drop(held); - completion.recv_timeout(Duration::from_secs(5)).unwrap(); - owner.join().unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::close_waits_for_publication_and_rejects_late_senders" + ), + move || { + let (sender, receiver) = threads::channel(); + let held = sender.admission.enter().unwrap(); + let (done, completion) = mpsc::channel(); + let owner = std::thread::spawn(move || { + drop(receiver); + done.send(()).unwrap(); + }); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !sender.admission.gate.is_closed() { + assert!(std::time::Instant::now() < deadline, "close did not start"); + std::thread::yield_now(); + } + assert!(sender.send(1).is_err()); + assert!(completion.try_recv().is_err()); + // Model a producer paused between reservation and raw publication. + sender.inner.send(2).unwrap(); + drop(held); + completion.recv_timeout(Duration::from_secs(5)).unwrap(); + owner.join().unwrap(); + }, + ); } #[cfg(feature = "async")] #[tokio::test] async fn compact_async_senders_close_after_their_last_owner() { - let (send, mut receive) = tasks::channel(); - let other = send.clone(); - send.send(1).unwrap(); - drop(send); - assert_eq!(receive.recv().await, Some(1)); - other.send(2).unwrap(); - drop(other); - assert_eq!(receive.recv().await, Some(2)); - assert_eq!( - tokio::time::timeout(Duration::from_secs(5), receive.recv()) - .await - .unwrap(), - None - ); + crate::test_wait::within( + concat!( + module_path!(), + "::compact_async_senders_close_after_their_last_owner" + ), + async move { + let (send, mut receive) = tasks::channel(); + let other = send.clone(); + send.send(1).unwrap(); + drop(send); + assert_eq!(receive.recv().await, Some(1)); + other.send(2).unwrap(); + drop(other); + assert_eq!(receive.recv().await, Some(2)); + assert_eq!( + tokio::time::timeout(Duration::from_secs(5), receive.recv()) + .await + .unwrap(), + None + ); + }, + ) + .await; } #[cfg(feature = "async")] #[tokio::test] async fn unpolled_tracked_unwind_drains_before_releasing_lease() { - struct State; - impl Drop for State { - fn drop(&mut self) { - panic!("state destructor"); - } - } - struct Bomb(Arc); - impl Drop for Bomb { - fn drop(&mut self) { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("queued destructor"); - } - } - let system = crate::system::ActorSystem::new().unwrap(); - let group = system.async_group("unpolled-tracked-drop", 1).unwrap(); - let (scope, lease) = group.reserve(None).unwrap(); - let (sender, receiver) = tasks::channel(); - let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { - assert!(sender.send(Bomb(dropped.clone())).is_ok()); - } - let payload = (State, receiver); - let future = async move { - drop(payload); - }; - let tracked = crate::system::Tracked { future, lease }; - assert!(catch_unwind(AssertUnwindSafe(|| drop(tracked))).is_err()); - assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!( - sender.failures().first().as_deref(), - Some("queued destructor") - ); - tokio::time::timeout(Duration::from_secs(5), scope.joined()) - .await - .unwrap(); - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::unpolled_tracked_unwind_drains_before_releasing_lease" + ), + async move { + struct State; + impl Drop for State { + fn drop(&mut self) { + panic!("state destructor"); + } + } + struct Bomb(Arc); + impl Drop for Bomb { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("queued destructor"); + } + } + let system = crate::system::ActorSystem::new().unwrap(); + let group = system.async_group("unpolled-tracked-drop", 1).unwrap(); + let (scope, lease) = group.reserve(None).unwrap(); + let (sender, receiver) = tasks::channel(); + let dropped = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + assert!(sender.send(Bomb(dropped.clone())).is_ok()); + } + let payload = (State, receiver); + let future = async move { + drop(payload); + }; + let tracked = crate::system::Tracked { future, lease }; + assert!(catch_unwind(AssertUnwindSafe(|| drop(tracked))).is_err()); + assert_eq!(dropped.load(Ordering::SeqCst), 2); + assert_eq!( + sender.failures().first().as_deref(), + Some("queued destructor") + ); + tokio::time::timeout(Duration::from_secs(5), scope.joined()) + .await + .unwrap(); + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, + ) + .await; } } diff --git a/concurrency/src/metrics.rs b/concurrency/src/metrics.rs index 3173516bf..5c15904ec 100644 --- a/concurrency/src/metrics.rs +++ b/concurrency/src/metrics.rs @@ -6,7 +6,7 @@ //! //! ``` //! use spawned_concurrency::metrics::{self, Config}; -//! metrics::configure(Config { timing_sample_every: 16 }).unwrap(); +//! metrics::configure(Config::new().timing_sample_every(16)).unwrap(); //! let snapshot = metrics::snapshot(); //! assert_eq!(snapshot.actors_active, 0); //! let text = metrics::render_prometheus(); @@ -25,7 +25,9 @@ use spawned_rt::time::Instant; /// Timing samples are deterministic, one in every N observations per category. /// Zero disables clocks/histograms while retaining counters and gauges. +/// Non-exhaustive: `Config::new().timing_sample_every(n)`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub struct Config { pub timing_sample_every: u64, } @@ -36,6 +38,18 @@ impl Default for Config { } } } +impl Config { + /// Time every observation (`timing_sample_every = 1`). + pub fn new() -> Self { + Self::default() + } + /// Time one in every `n` observations per category; 0 keeps counters + /// and gauges only. + pub fn timing_sample_every(mut self, n: u64) -> Self { + self.timing_sample_every = n; + self + } +} struct SamplingPolicy { // Only explicit requirements compose. Implicit default instrumentation must // not prevent an embedded component from configuring sampling later. @@ -109,6 +123,7 @@ pub const BUCKET_BOUNDS_NS: [u64; 32] = { bounds }; #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub struct HistogramSnapshot { pub buckets: [u64; 32], pub count: u64, @@ -159,8 +174,9 @@ macro_rules! counters { /// Process aggregates. Reads are independent atomic samples, not a /// transaction. Conservation equations hold after operations quiesce. #[derive(Debug, Clone)] - pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot } - pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot() } } + #[non_exhaustive] + pub struct Snapshot { $( pub $name: u64, )* pub queue_wait: HistogramSnapshot, pub handler_duration: HistogramSnapshot, pub request_latency: HistogramSnapshot, pub deferred_reply_latency: HistogramSnapshot } + pub fn snapshot() -> Snapshot { Snapshot { $( $name: COUNTERS.$name.load(Relaxed), )* queue_wait: QUEUE_WAIT.snapshot(), handler_duration: HANDLER_DURATION.snapshot(), request_latency: REQUEST_LATENCY.snapshot(), deferred_reply_latency: DEFERRED_REPLY_LATENCY.snapshot() } } fn render_counters(out: &mut String, s: &Snapshot) { $( let name = concat!("spawned_", stringify!($name)); @@ -177,6 +193,8 @@ counters!( actors_exited_shutdown_total, actors_exited_panic_total, actors_exited_kill_total, + actors_exited_killed_total, + actors_exited_noproc_total, messages_enqueued_total, messages_rejected_total, mailbox_depth, @@ -193,11 +211,34 @@ counters!( requests_replied_total, requests_timed_out_total, requests_actor_stopped_total, + requests_noproc_total, + requests_reply_dropped_total, + requests_overloaded_total, requests_cancelled_total, + deferred_tokens_total, + deferred_pending, + deferred_replied_total, + deferred_expired_total, + deferred_dropped_total, + deferred_stopped_total, + deferred_abandoned_total, + deferred_overloaded_total, + jobs_started_total, + jobs_running, + jobs_completed_total, + jobs_panicked_total, + jobs_deadline_exceeded_total, + jobs_cancelled_total, + jobs_fenced_total, + jobs_rejected_total, + jobs_emits_total, + jobs_unfinished_at_stop_total, ); static QUEUE_WAIT: Histogram = Histogram::new(); static HANDLER_DURATION: Histogram = Histogram::new(); static REQUEST_LATENCY: Histogram = Histogram::new(); +static DEFERRED_REPLY_LATENCY: Histogram = Histogram::new(); +static DEFERRED_SEQUENCE: AtomicU64 = AtomicU64::new(0); static MESSAGE_SEQUENCE: AtomicU64 = AtomicU64::new(0); static REQUEST_SEQUENCE: AtomicU64 = AtomicU64::new(0); @@ -214,6 +255,10 @@ pub fn render_prometheus() -> String { &snapshot.handler_duration, ), ("spawned_request_latency_seconds", &snapshot.request_latency), + ( + "spawned_deferred_reply_latency_seconds", + &snapshot.deferred_reply_latency, + ), ] { writeln!(out, "# TYPE {name} histogram").unwrap(); let mut count = 0; @@ -243,6 +288,85 @@ pub fn render_prometheus() -> String { out } +/// Deferred-reply and pipe-to-self job accounting +/// (docs/design/deferred-replies.md; `spawned_deferred_*`, `spawned_jobs_*`). +pub(crate) mod hooks { + use super::{sample, Instant, COUNTERS, DEFERRED_REPLY_LATENCY, DEFERRED_SEQUENCE}; + use std::sync::atomic::Ordering::Relaxed; + + /// When a token was deferred, for a sampled reply latency. + pub(crate) type DeferStamp = Option; + + /// `defer()` took a credit: the slot now has a live token. + pub(crate) fn deferred() -> DeferStamp { + COUNTERS.deferred_tokens_total.fetch_add(1, Relaxed); + COUNTERS.deferred_pending.fetch_add(1, Relaxed); + sample(&DEFERRED_SEQUENCE) + } + + /// A deferred token delivered its reply. + pub(crate) fn deferred_replied(at: &DeferStamp) { + if let Some(at) = at { + DEFERRED_REPLY_LATENCY.record(at.elapsed()); + } + } + + /// A slot left `OPEN` for `kind` (the model's terminal name); `token` + /// tells whether a deferred token was live. + pub(crate) fn left_open(kind: &'static str, token: bool) { + if kind == "overloaded" { + COUNTERS.deferred_overloaded_total.fetch_add(1, Relaxed); + } + if !token { + return; + } + COUNTERS.deferred_pending.fetch_sub(1, Relaxed); + match kind { + "ok" => &COUNTERS.deferred_replied_total, + "timeout" => &COUNTERS.deferred_expired_total, + "dropped" => &COUNTERS.deferred_dropped_total, + "stopped" => &COUNTERS.deferred_stopped_total, + "abandoned" => &COUNTERS.deferred_abandoned_total, + _ => return, + } + .fetch_add(1, Relaxed); + } + + pub(crate) fn job_started() { + COUNTERS.jobs_started_total.fetch_add(1, Relaxed); + COUNTERS.jobs_running.fetch_add(1, Relaxed); + } + pub(crate) fn job_ended() { + COUNTERS.jobs_running.fetch_sub(1, Relaxed); + } + /// The terminal notification was taken: `ok`, `panicked` or `deadline`. + pub(crate) fn job_notified(end: &'static str) { + match end { + "ok" => &COUNTERS.jobs_completed_total, + "panicked" => &COUNTERS.jobs_panicked_total, + _ => &COUNTERS.jobs_deadline_exceeded_total, + } + .fetch_add(1, Relaxed); + } + pub(crate) fn job_cancelled() { + COUNTERS.jobs_cancelled_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_fenced() { + COUNTERS.jobs_fenced_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_rejected() { + COUNTERS.jobs_rejected_total.fetch_add(1, Relaxed); + } + pub(crate) fn job_emitted() { + COUNTERS.jobs_emits_total.fetch_add(1, Relaxed); + } + pub(crate) fn jobs_unfinished(n: usize) { + COUNTERS + .jobs_unfinished_at_stop_total + .fetch_add(n as u64, Relaxed); + } +} + #[derive(Clone, Default)] pub(crate) struct Mailbox(Arc); impl Mailbox { @@ -361,6 +485,8 @@ impl ActorObservation { ExitReason::Shutdown => &COUNTERS.actors_exited_shutdown_total, ExitReason::Panic(_) => &COUNTERS.actors_exited_panic_total, ExitReason::Kill => &COUNTERS.actors_exited_kill_total, + ExitReason::Killed => &COUNTERS.actors_exited_killed_total, + ExitReason::NoProc => &COUNTERS.actors_exited_noproc_total, } .fetch_add(1, Relaxed); } @@ -368,7 +494,7 @@ impl ActorObservation { impl Drop for ActorObservation { fn drop(&mut self) { if !self.finished { - self.finish(&ExitReason::Kill); + self.finish(&ExitReason::Killed); } } } @@ -390,7 +516,10 @@ impl RequestObservation { match result { Ok(_) => &COUNTERS.requests_replied_total, Err(ActorError::RequestTimeout) => &COUNTERS.requests_timed_out_total, - Err(ActorError::ActorStopped) => &COUNTERS.requests_actor_stopped_total, + Err(ActorError::ActorStopped(_)) => &COUNTERS.requests_actor_stopped_total, + Err(ActorError::NoProc) => &COUNTERS.requests_noproc_total, + Err(ActorError::ReplyDropped) => &COUNTERS.requests_reply_dropped_total, + Err(ActorError::Overloaded) => &COUNTERS.requests_overloaded_total, } .fetch_add(1, Relaxed); } diff --git a/concurrency/src/monitor.rs b/concurrency/src/monitor.rs index 578f835c8..aeef96622 100644 --- a/concurrency/src/monitor.rs +++ b/concurrency/src/monitor.rs @@ -23,6 +23,10 @@ impl MonitorRef { pub(crate) fn next() -> Self { Self(NEXT_MONITOR_ID.fetch_add(1, Ordering::Relaxed)) } + #[allow(dead_code)] // used by lifecycle tracing only + pub(crate) fn raw(self) -> u64 { + self.0 + } } impl std::fmt::Display for MonitorRef { @@ -35,41 +39,57 @@ impl std::fmt::Display for MonitorRef { // Down // --------------------------------------------------------------------------- -/// Message delivered to a monitoring actor when its target stops. +/// Message delivered to a monitoring actor when its target stops: OTP's +/// `{'DOWN', Ref, process, Object, Info}`. /// /// To monitor another actor, call `ctx.monitor(&child_handle)` and implement -/// `Handler` on the monitoring actor. +/// `Handler` on the monitoring actor. Non-exhaustive: match it with +/// `Down { monitor_ref, actor, reason, .. }`. #[derive(Debug, Clone, PartialEq)] +#[non_exhaustive] pub struct Down { /// The monitor that triggered this notification. pub monitor_ref: MonitorRef, + /// The monitored actor (OTP's `Object`). + pub actor: crate::ActorId, /// Why the monitored actor stopped. pub reason: ExitReason, } +impl Down { + /// A down notice for monitor `monitor_ref` of `actor` (for test doubles + /// and comparisons; the framework builds the ones it delivers). + pub fn new(monitor_ref: MonitorRef, actor: crate::ActorId, reason: ExitReason) -> Self { + Self { + monitor_ref, + actor, + reason, + } + } +} + impl Message for Down { type Result = (); } +mod sync { + pub(crate) use std::sync::{Mutex, MutexGuard}; +} +pub(crate) mod fence; + /// Each monitor owns a cancellation token that wakes its waiter immediately. -pub(crate) type MonitorTable = std::sync::Arc< - std::sync::Mutex< - Option>, - >, ->; +pub(crate) type MonitorTable = + std::sync::Arc>; /// Release outgoing monitors on every actor exit path, including unwinding. pub(crate) struct MonitorCleanup(pub MonitorTable); impl Drop for MonitorCleanup { fn drop(&mut self) { - let monitors = self.0.lock().unwrap_or_else(|p| p.into_inner()).take(); - if let Some(monitors) = monitors { - // HashMap order is per-process random; under simulation, wake - // waiters in monitor-creation order so the schedule is seeded. - #[cfg(spawned_sim)] - let monitors: std::collections::BTreeMap = - monitors.into_iter().map(|(r, t)| (r.0, t)).collect(); - for token in monitors.into_values() { + if let Some(mut monitors) = fence::close(&self.0) { + // HashMap order is per-process random: wake waiters in + // monitor-creation order, so a simulated schedule is seeded. + monitors.sort_unstable_by_key(|(r, _)| r.0); + for (_, token) in monitors { token.cancel(); } } @@ -80,9 +100,7 @@ impl Drop for MonitorCleanup { pub(crate) struct MonitorRegistration(pub MonitorTable, pub MonitorRef); impl Drop for MonitorRegistration { fn drop(&mut self) { - if let Some(entries) = self.0.lock().unwrap_or_else(|p| p.into_inner()).as_mut() { - entries.remove(&self.1); - } + drop(fence::remove(&self.0, &self.1, || {})); } } @@ -104,8 +122,9 @@ mod tests { let table: MonitorTable = Default::default(); let (mine, other) = (MonitorRef::next(), MonitorRef::next()); let mut entries = std::collections::HashMap::new(); - entries.insert(mine, spawned_rt::sync::CancellationToken::new()); - entries.insert(other, spawned_rt::sync::CancellationToken::new()); + let token = spawned_rt::sync::CancellationToken::new; + entries.insert(mine, fence::Entry::Active(token())); + entries.insert(other, fence::Entry::Active(token())); *table.lock().unwrap() = Some(entries); drop(MonitorRegistration(table.clone(), mine)); let entries = table.lock().unwrap(); @@ -114,6 +133,30 @@ mod tests { assert!(entries.contains_key(&other)); } + /// A finished waiter's registration drop leaves its queued `Down` for + /// dispatch; `demonitor` flushes it, and dispatch then skips it. + #[test] + fn demonitor_flushes_a_queued_down() { + let table: MonitorTable = std::sync::Arc::new(fence::open()); + for flush in [false, true] { + let monitor = MonitorRef::next(); + let token = spawned_rt::sync::CancellationToken::new(); + assert_eq!( + fence::register(&table, monitor, token, || false, || false, || {}, || {}), + fence::Registration::Active + ); + let mut enqueued = false; + assert!(fence::deliver(&table, &monitor, || enqueued = true)); + assert!(enqueued); + drop(MonitorRegistration(table.clone(), monitor)); + if flush { + assert!(fence::demonitor(&table, &monitor, || {}).is_none()); + } + assert_eq!(fence::take_queued(&table, &monitor), !flush); + assert!(!fence::take_queued(&table, &monitor), "taken once"); + } + } + #[test] fn registration_dropped_after_cleanup_is_harmless() { let table: MonitorTable = Default::default(); diff --git a/concurrency/src/monitor/fence.rs b/concurrency/src/monitor/fence.rs new file mode 100644 index 000000000..f7bbaa27e --- /dev/null +++ b/concurrency/src/monitor/fence.rs @@ -0,0 +1,175 @@ +//! The observer-side monitor table protocol (docs/design/otp-exit-signals.md +//! E4, E6): registration linearised against the target's death point, the +//! `Down` enqueued in the same critical section that marks the entry +//! queued, the `demonitor` fence and flush (OTP `demonitor(Ref, [flush])`) +//! and the observer's own close. +//! +//! Shared with the Loom model (`tests/link_fence_loom.rs`), which supplies +//! the synchronization types through `super::sync`. The callbacks run under +//! the observer's table lock; production uses them only to read the target's +//! death point, record a trace event and enqueue a `Down` (which never takes +//! a monitor table lock). + +use super::sync::{Mutex, MutexGuard}; +use std::collections::HashMap; +use std::hash::Hash; + +/// One monitor in its observer's table. +#[derive(Debug)] +pub(crate) enum Entry { + /// Registered: its waiter delivers the `Down` at the target's + /// publication. + Active(V), + /// Its `Down` is in the observer's mailbox, not yet handled: dispatch + /// takes it ([`take_queued`]); `demonitor` flushes it. + Queued, +} + +/// An observer's monitors: `None` once the observer's exit closed it. +pub(crate) type Entries = Mutex>>>; + +fn lock(table: &Entries) -> MutexGuard<'_, Option>>> { + table.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// An open, empty table. +pub(crate) fn open() -> Entries { + Mutex::new(Some(HashMap::new())) +} + +/// The outcome of a `monitor` call. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Registration { + /// The observer is exiting or closed: no entry, no notice. + Inert, + /// The target was past D: the `NoProc` notice was enqueued (its entry is + /// `Queued`). + Late, + /// The entry is installed; its waiter delivers the `Down` at publication. + Active, +} + +/// `monitor`: under the observer's lock, inert once `observer_exiting` or the +/// table is closed; past the target's D (`target_dead`, which reads the +/// target's exit table) enqueue the late notice (`late`) and mark it queued; +/// otherwise install the entry and run `active`. +pub(crate) fn register( + table: &Entries, + key: K, + value: V, + observer_exiting: impl FnOnce() -> bool, + target_dead: impl FnOnce() -> bool, + late: impl FnOnce(), + active: impl FnOnce(), +) -> Registration { + let mut guard = lock(table); + if observer_exiting() { + return Registration::Inert; + } + let Some(entries) = guard.as_mut() else { + return Registration::Inert; + }; + if target_dead() { + late(); + entries.insert(key, Entry::Queued); + return Registration::Late; + } + entries.insert(key, Entry::Active(value)); + active(); + Registration::Active +} + +/// The waiter, after the target's exit is published: mark the entry queued +/// and enqueue the `Down` (`deliver`) in one critical section, so once +/// [`demonitor`] returns no `Down` of this monitor is enqueued, and one +/// already enqueued is flushed. Returns whether it was delivered. +pub(crate) fn deliver( + table: &Entries, + key: &K, + deliver: impl FnOnce(), +) -> bool { + let mut guard = lock(table); + let entry = guard.as_mut().and_then(|entries| entries.get_mut(key)); + let active = match entry { + Some(entry @ Entry::Active(_)) => { + *entry = Entry::Queued; + true + } + _ => false, + }; + if active { + deliver(); + } + drop(guard); + active +} + +/// Dispatch of a framework `Down`: whether it is still owed (its entry is +/// `Queued`; taken now). `false` once `demonitor` flushed it. +pub(crate) fn take_queued(table: &Entries, key: &K) -> bool { + let mut guard = lock(table); + let Some(entries) = guard.as_mut() else { + return false; + }; + if matches!(entries.get(key), Some(Entry::Queued)) { + entries.remove(key); + true + } else { + false + } +} + +/// A waiter that is dropped (unpolled, rejected or finished): remove an +/// `Active` entry under the lock, running `removed` there when it was +/// present. A `Queued` entry stays for its dispatch. The caller wakes the +/// waiter with the returned value after the lock is released. +pub(crate) fn remove( + table: &Entries, + key: &K, + removed: impl FnOnce(), +) -> Option { + let mut guard = lock(table); + let entries = guard.as_mut()?; + if !matches!(entries.get(key), Some(Entry::Active(_))) { + return None; + } + removed(); + match entries.remove(key) { + Some(Entry::Active(value)) => Some(value), + _ => None, + } +} + +/// `demonitor` with OTP's `flush`: under the lock, remove the entry +/// whatever its state, running `removed` there when it was present. An +/// `Active` entry's waiter (returned) is woken by the caller after the lock +/// is released; a `Queued` one's `Down`, already in the mailbox, will be +/// skipped at dispatch. Once this returns, no `Down` of the monitor is +/// enqueued or handled. +pub(crate) fn demonitor( + table: &Entries, + key: &K, + removed: impl FnOnce(), +) -> Option { + let mut guard = lock(table); + let entry = guard.as_mut().and_then(|entries| entries.remove(key))?; + removed(); + match entry { + Entry::Active(value) => Some(value), + Entry::Queued => None, + } +} + +/// The observer's exit: close the table and take every active entry, whose +/// waiters the caller wakes after the lock is released. +pub(crate) fn close(table: &Entries) -> Option> { + lock(table).take().map(|entries| { + entries + .into_iter() + .filter_map(|(key, entry)| match entry { + Entry::Active(value) => Some((key, value)), + Entry::Queued => None, + }) + .collect() + }) +} diff --git a/concurrency/src/native_link_regression_tests.rs b/concurrency/src/native_link_regression_tests.rs index 078902613..6fbc625de 100644 --- a/concurrency/src/native_link_regression_tests.rs +++ b/concurrency/src/native_link_regression_tests.rs @@ -2,38 +2,46 @@ use crate::{link, ActorId, ExitReason}; use std::time::Duration; #[test] fn start_linked_registers_before_threads_started() { - use crate::threads::{ - Actor as ThreadActor, ActorStart as ThreadStart, Context as ThreadContext, - }; - struct Parent; - impl ThreadActor for Parent {} - struct Child { - parent: ActorId, - seen: std::sync::mpsc::Sender, - } - impl ThreadActor for Child { - fn started(&mut self, ctx: &ThreadContext) { - let h = ctx.actor_ref().child_handle(); - self.seen - .send(link::has_link(h.links(), self.parent)) - .unwrap(); - ctx.stop(); - } - } - let parent = Parent.start(); - for _ in 0..32 { - let (tx, rx) = std::sync::mpsc::channel(); - let child = Child { - parent: parent.id(), - seen: tx, - } - .start_linked(&parent.context()); - assert!(rx.recv_timeout(Duration::from_secs(1)).unwrap()); - assert_eq!( - child.child_handle().wait_exit_blocking(), - ExitReason::Normal - ); - } - parent.child_handle().stop(); - parent.child_handle().wait_exit_blocking(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::start_linked_registers_before_threads_started" + ), + move || { + use crate::threads::{ + Actor as ThreadActor, ActorStart as ThreadStart, Context as ThreadContext, + }; + struct Parent; + impl ThreadActor for Parent {} + struct Child { + parent: ActorId, + seen: std::sync::mpsc::Sender, + } + impl ThreadActor for Child { + fn started(&mut self, ctx: &ThreadContext) { + let h = ctx.actor_ref().child_handle(); + self.seen + .send(link::has_link(h.links(), self.parent)) + .unwrap(); + ctx.stop(); + } + } + let parent = Parent.start(); + for _ in 0..32 { + let (tx, rx) = std::sync::mpsc::channel(); + let child = Child { + parent: parent.id(), + seen: tx, + } + .start_linked(&parent.context()); + assert!(rx.recv_timeout(Duration::from_secs(1)).unwrap()); + assert_eq!( + child.child_handle().wait_exit_blocking(), + ExitReason::Normal + ); + } + parent.child_handle().stop(); + parent.child_handle().wait_exit_blocking(); + }, + ); } diff --git a/concurrency/src/registry.rs b/concurrency/src/registry.rs index eeb55405b..c9ec1d6be 100644 --- a/concurrency/src/registry.rs +++ b/concurrency/src/registry.rs @@ -17,6 +17,7 @@ fn global_store() -> &'static Store { /// Errors that can occur when registering a value in the registry. #[derive(Debug, thiserror::Error)] +#[non_exhaustive] pub enum RegistryError { /// A value with this name is already registered. #[error("name '{0}' is already registered")] diff --git a/concurrency/src/reply/deadline.rs b/concurrency/src/reply/deadline.rs new file mode 100644 index 000000000..1bd7bea0d --- /dev/null +++ b/concurrency/src/reply/deadline.rs @@ -0,0 +1,370 @@ +//! Per-actor deadline heaps and the deadline service +//! (docs/design/deferred-replies.md ยง2, "Deadline storage"). +//! +//! Each actor's registry owns a min-heap of `(deadline, entry)`, where an entry +//! is a deferred token's slot (and, with jobs, a job state). Entries hold +//! `Weak` references, so the heap owns nothing. An entry is stale when its +//! item is gone or terminal, or when its deadline is later than the item's +//! current one (superseded by `with_deadline`); the heap is rebuilt when stale +//! entries exceed live ones. +//! +//! The service holds one entry per actor, its earliest deadline, and expires +//! due entries even when no caller polls, so credits return on time: +//! +//! - real time: one process-wide service thread with its own timer, so +//! per-test Tokio runtimes cannot break it; +//! - under `spawned_sim`: a task of the simulated runtime per actor heap, so +//! clock domains never mix and virtual time drives it. + +use super::registry::Registry; +use std::cmp::Reverse; +use std::collections::BinaryHeap; +use std::sync::{Arc, Weak}; + +/// Something with a deadline the owner's heap services. +pub(crate) trait Expirable: Send + Sync { + /// The current deadline while the item is pending, else `None`. + fn pending_deadline(&self) -> Option; + /// The deadline passed (`now >= deadline`): expire the item. + fn expire(&self, now: u64); +} + +struct Entry { + at: u64, + seq: u64, + item: Weak, +} +impl PartialEq for Entry { + fn eq(&self, other: &Self) -> bool { + (self.at, self.seq) == (other.at, other.seq) + } +} +impl Eq for Entry {} +impl PartialOrd for Entry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for Entry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + (self.at, self.seq).cmp(&(other.at, other.seq)) + } +} + +impl Entry { + /// Live: the item exists, is pending, and this is its current deadline. + /// The upgraded item is handed to `upgraded`, to be dropped by the caller + /// outside every lock (it may be the last strong reference). + fn is_live(&self, upgraded: &mut Vec>) -> bool { + let Some(item) = self.item.upgrade() else { + return false; + }; + let live = item.pending_deadline() == Some(self.at); + upgraded.push(item); + live + } +} + +/// One actor's deadline heap. +#[derive(Default)] +pub(crate) struct DeadlineHeap { + heap: BinaryHeap>, + seq: u64, + /// Live entries at the last rebuild; the heap is rebuilt when it holds + /// more than twice that many (plus slack). + live_at_rebuild: usize, +} + +impl DeadlineHeap { + /// Add an entry. Returns whether it is the new earliest deadline, and + /// the items compaction upgraded (drop them outside the lock). + #[must_use] + pub(crate) fn push( + &mut self, + at: u64, + item: Weak, + ) -> (bool, Vec>) { + let mut upgraded = Vec::new(); + if self.heap.len() >= 2 * self.live_at_rebuild + 16 { + self.compact(&mut upgraded); + } + self.seq += 1; + let earliest = self.earliest().is_none_or(|e| at < e); + self.heap.push(Reverse(Entry { + at, + seq: self.seq, + item, + })); + (earliest, upgraded) + } + + pub(crate) fn earliest(&self) -> Option { + self.heap.peek().map(|Reverse(e)| e.at) + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.heap.len() + } + + /// Rebuild without stale entries. + fn compact(&mut self, upgraded: &mut Vec>) { + let entries = std::mem::take(&mut self.heap).into_vec(); + self.heap = entries + .into_iter() + .filter(|Reverse(e)| e.is_live(upgraded)) + .collect(); + self.live_at_rebuild = self.heap.len(); + } + + /// Pop every entry due at `now`; the caller expires them outside the lock. + pub(crate) fn due(&mut self, now: u64) -> Vec> { + let mut due = Vec::new(); + while let Some(Reverse(top)) = self.heap.peek() { + if top.at > now { + break; + } + let Reverse(entry) = self.heap.pop().expect("peeked"); + if let Some(item) = entry.item.upgrade() { + due.push(item); + } + } + due + } + + /// Drop every entry (the owner died). + pub(crate) fn clear(&mut self) { + self.heap.clear(); + } +} + +// --------------------------------------------------------------------------- +// Service +// --------------------------------------------------------------------------- + +/// The owner died: the service forgets it (its heap is gone). +pub(crate) fn forget(_registry: &Registry) { + #[cfg(not(all(spawned_sim, feature = "async")))] + thread_service::forget(_registry.key()); +} + +/// Ask the service to visit `registry` at `at`. +pub(crate) fn schedule(registry: &Arc, at: u64) { + #[cfg(all(spawned_sim, feature = "async"))] + sim::schedule(registry, at); + #[cfg(not(all(spawned_sim, feature = "async")))] + thread_service::schedule(registry, at); +} + +#[cfg(not(all(spawned_sim, feature = "async")))] +mod thread_service { + use super::*; + use std::collections::HashMap; + use std::sync::{Condvar, Mutex, OnceLock}; + use std::time::Duration; + + #[derive(Default)] + struct State { + /// `(at, key)`; an entry is current only if `actors[key].0 == at`. + queue: BinaryHeap>, + actors: HashMap)>, + } + + struct Service { + state: Mutex, + wake: Condvar, + } + + fn lock(service: &Service) -> std::sync::MutexGuard<'_, State> { + service.state.lock().unwrap_or_else(|p| p.into_inner()) + } + + fn service() -> &'static Service { + static SERVICE: OnceLock<&'static Service> = OnceLock::new(); + SERVICE.get_or_init(|| { + let service: &'static Service = Box::leak(Box::new(Service { + state: Mutex::new(State::default()), + wake: Condvar::new(), + })); + std::thread::Builder::new() + .name("spawned-deadlines".into()) + .spawn(move || run(service)) + .expect("spawn the deadline service thread"); + service + }) + } + + pub(super) fn forget(key: u64) { + let Some(service) = SERVICE_REF.get() else { + return; + }; + let forgotten = lock(service).actors.remove(&key); + drop(forgotten); + } + + static SERVICE_REF: OnceLock<&'static Service> = OnceLock::new(); + + pub(super) fn schedule(registry: &Arc, at: u64) { + let service = service(); + let _ = SERVICE_REF.set(service); + let key = registry.key(); + let mut state = lock(service); + match state.actors.get(&key) { + Some((current, _)) if *current <= at => return, + _ => {} + } + state.actors.insert(key, (at, Arc::downgrade(registry))); + state.queue.push(Reverse((at, key))); + if state.queue.len() > 2 * state.actors.len() + 64 { + let actors = &state.actors; + let live: BinaryHeap<_> = actors + .iter() + .map(|(key, (at, _))| Reverse((*at, *key))) + .collect(); + state.queue = live; + } + drop(state); + service.wake.notify_one(); + } + + fn run(service: &'static Service) { + let mut state = lock(service); + loop { + let now = super::super::now_ticks(); + match state.queue.peek().copied() { + None => { + state = service.wake.wait(state).unwrap_or_else(|p| p.into_inner()); + } + Some(Reverse((at, key))) if at <= now => { + state.queue.pop(); + let current = state.actors.get(&key).map(|(a, w)| (*a, w.clone())); + let Some((current_at, weak)) = current else { + continue; + }; + if current_at != at { + continue; // superseded by an earlier schedule + } + state.actors.remove(&key); + drop(state); + // Entries are contained one by one; this also covers the + // visit itself and dropping the upgraded registry, so the + // one process-wide service never stops. + super::super::registry::contain_entry(|| { + if let Some(registry) = weak.upgrade() { + if let Some(next) = registry.service_deadlines(now) { + schedule(®istry, next); + } + } + }); + state = lock(service); + } + Some(Reverse((at, _))) => { + let wait = Duration::from_nanos(at - now); + state = service + .wake + .wait_timeout(state, wait) + .unwrap_or_else(|p| p.into_inner()) + .0; + } + } + } + } +} + +#[cfg(all(spawned_sim, feature = "async"))] +mod sim { + use super::*; + use futures::task::AtomicWaker; + use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; + use std::task::Poll; + use std::time::Duration; + + /// Per-heap service task state, kept by the registry. + pub(crate) struct SimService { + started: AtomicBool, + earliest: AtomicU64, + waker: AtomicWaker, + } + + impl Default for SimService { + fn default() -> Self { + Self { + started: AtomicBool::new(false), + // Nothing scheduled yet: a first schedule only lowers it, so a + // concurrent earlier deadline is never overwritten. + earliest: AtomicU64::new(u64::MAX), + waker: AtomicWaker::new(), + } + } + } + + pub(super) fn schedule(registry: &Arc, at: u64) { + let service = registry.sim_service(); + service.earliest.fetch_min(at, Ordering::AcqRel); + if !service.started.swap(true, Ordering::AcqRel) { + let weak = Arc::downgrade(registry); + spawned_rt::tasks::spawn(run(weak)); + } + service.waker.wake(); + } + + async fn run(weak: Weak) { + loop { + let Some(registry) = weak.upgrade() else { + return; + }; + if registry.is_dead() { + return; + } + let at = registry.sim_service().earliest.load(Ordering::Acquire); + drop(registry); + let now = super::super::now_ticks(); + if at <= now { + let Some(registry) = weak.upgrade() else { + return; + }; + registry + .sim_service() + .earliest + .store(u64::MAX, Ordering::Release); + if let Some(next) = registry.service_deadlines(now) { + registry + .sim_service() + .earliest + .fetch_min(next, Ordering::AcqRel); + } + continue; + } + // Sleep until `at`, or until an earlier deadline or the death + // wakes us. + let mut sleep = (at != u64::MAX) + .then(|| Box::pin(spawned_rt::tasks::sleep(Duration::from_nanos(at - now)))); + let weak_poll = weak.clone(); + std::future::poll_fn(|cx| { + let Some(registry) = weak_poll.upgrade() else { + return Poll::Ready(()); + }; + let service = registry.sim_service(); + service.waker.register(cx.waker()); + if registry.is_dead() || service.earliest.load(Ordering::Acquire) < at { + return Poll::Ready(()); + } + match &mut sleep { + Some(sleep) => sleep.as_mut().poll(cx), + None => Poll::Pending, + } + }) + .await; + } + } + + impl SimService { + pub(crate) fn wake(&self) { + self.waker.wake(); + } + } + use std::future::Future; +} + +#[cfg(all(spawned_sim, feature = "async"))] +pub(crate) use sim::SimService; diff --git a/concurrency/src/reply/job_core.rs b/concurrency/src/reply/job_core.rs new file mode 100644 index 000000000..02bfc8372 --- /dev/null +++ b/concurrency/src/reply/job_core.rs @@ -0,0 +1,544 @@ +//! The racy core of pipe-to-self jobs (docs/design/deferred-replies.md ยง8, +//! ยง9; `verification/tla/DeferredJobs.tla`): one job's cancellation flag, +//! one-shot terminal notification and emit credits, and the owner's job set +//! (key map, running count, native budget). +//! +//! Shared with the Loom model (`tests/job_core_loom.rs`), which supplies the +//! synchronization types through `super::sync`, including a model of +//! `Arc`/`Weak` that observes the final destruction of a job. The protocol +//! never runs user code under its locks: `enqueue` closures only push an +//! envelope, and every strong reference a lock section takes (a replaced key, +//! an upgraded `Weak`) is dropped after the lock is released. + +use super::sync::{ + AtomicBool, AtomicUsize, Condvar, JobArc as Arc, JobWeak as Weak, Mutex, MutexGuard, Ordering, +}; +use std::collections::HashMap; +use std::hash::Hash; + +/// The job is cancelled or already notified: an emit was not enqueued. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("the job is gone (cancelled, replaced or past its terminal notification)")] +pub struct JobGone; + +/// The terminal notification: a one-shot, taken by the result, a caught +/// panic or the job deadline, and suppressed by cancellation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Notification { + Pending, + Notified, + Suppressed, +} + +/// A terminal notification held back until the job's scheduled emits have +/// fired (`JobCtx::emit_at`); run outside the lock. +pub(crate) type HeldTerminal = Box; + +struct Inner { + cancelled: bool, + notification: Notification, + running: bool, + emit_credits: u16, + /// Delayed emits (`emit_at`) holding a credit, not yet fired. + scheduled: u16, + /// The work's result, waiting for `scheduled` to reach 0 so the terminal + /// message stays last. + held: Option, +} + +impl Inner { + /// The terminal notification can still be taken. + fn open(&self) -> bool { + !self.cancelled && self.notification == Notification::Pending + } + /// New emits are accepted: open, and the work has not handed in its + /// result (a held terminal). + fn accepting(&self) -> bool { + self.open() && self.held.is_none() + } +} + +/// An emit's step 2: enqueued, or `JobGone` with the envelope the mailbox +/// refused (to drop outside locks). +pub(crate) type EmitResult = Result<(), (JobGone, Option)>; + +/// What [`JobCore::finish`] did with the work's result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Finish { + /// Notified now (the enqueue ran). + Taken, + /// Held until the scheduled emits fire. + Held, + /// Cancelled or already notified (deadline): nothing to deliver. + Refused, +} + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// One job's flag, notification and emit credits. +pub(crate) struct JobCore { + inner: Mutex, + emitters: Condvar, + cancelled_fast: AtomicBool, +} + +impl JobCore { + pub(crate) fn new(emit_credits: u16) -> Self { + Self { + inner: Mutex::new(Inner { + cancelled: false, + notification: Notification::Pending, + running: true, + emit_credits, + scheduled: 0, + held: None, + }), + emitters: Condvar::new(), + cancelled_fast: AtomicBool::new(false), + } + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.cancelled_fast.load(Ordering::Acquire) + } + + /// Set the cancellation flag: suppresses a pending notification and + /// wakes blocked emitters. Never undoes effects. Returns whether this + /// call set the flag. The caller wakes async emitters afterwards. + pub(crate) fn cancel(&self) -> bool { + let (newly, held) = self.cancel_keeping_held(); + // A held result is suppressed; it drops outside the lock. + drop(held); + newly + } + + /// [`cancel`](Self::cancel) that hands the suppressed held result back + /// instead of dropping it (user payload): the owner's exit destroys it + /// only after its death point answered every caller. + pub(crate) fn cancel_keeping_held(&self) -> (bool, Option) { + let (newly, held) = { + let mut inner = lock(&self.inner); + let newly = !inner.cancelled; + inner.cancelled = true; + if inner.notification == Notification::Pending { + inner.notification = Notification::Suppressed; + } + // Set under the lock, so a dispatch that sees an enqueued + // message's job as not cancelled ran before this cancel. + self.cancelled_fast.store(true, Ordering::Release); + (newly, inner.held.take()) + }; + self.emitters.notify_all(); + (newly, held) + } + + /// Take the terminal notification and run `enqueue` under the job's + /// lock. Returns whether it was taken; `enqueue` is not called when the + /// job was cancelled or already notified. Whatever `enqueue` hands back + /// (an envelope the mailbox refused) is returned to be dropped by the + /// caller after the lock is released. When taken, the caller wakes async + /// emitters. + pub(crate) fn notify(&self, enqueue: impl FnOnce() -> Option) -> (bool, Option) { + let (taken, unsent, held) = self.notify_keeping_held(enqueue); + // A deadline overtaking a held result: the result is dropped. + drop(held); + (taken, unsent) + } + + /// [`notify`](Self::notify) that hands a displaced held result back + /// (user payload) for the caller to drop outside its critical sections. + pub(crate) fn notify_keeping_held( + &self, + enqueue: impl FnOnce() -> Option, + ) -> (bool, Option, Option) { + let (unsent, held) = { + let mut inner = lock(&self.inner); + if !inner.open() { + return (false, None, None); + } + inner.notification = Notification::Notified; + (enqueue(), inner.held.take()) + }; + self.emitters.notify_all(); + (true, unsent, held) + } + + /// The work's result: take the notification as [`notify`](Self::notify) + /// does, unless scheduled emits are still pending; then store `later` + /// (which calls `notify` again) for the last of them to run, so the + /// terminal message stays last. + pub(crate) fn finish( + &self, + enqueue: impl FnOnce() -> Option, + later: impl FnOnce() -> HeldTerminal, + ) -> (Finish, Option) { + { + let mut inner = lock(&self.inner); + if !inner.open() { + return (Finish::Refused, None); + } + if inner.scheduled > 0 { + inner.held = Some(later()); + return (Finish::Held, None); + } + } + // Nothing scheduled, and only the work schedules: `notify` takes it + // unless a cancel or the deadline came in between. + match self.notify(enqueue) { + (true, unsent) => (Finish::Taken, unsent), + (false, _) => (Finish::Refused, None), + } + } + + /// Whether the terminal notification can still be taken (the job + /// deadline is still due). + pub(crate) fn is_pending(&self) -> bool { + lock(&self.inner).open() + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn notification(&self) -> Notification { + lock(&self.inner).notification + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn emit_credits(&self) -> u16 { + lock(&self.inner).emit_credits + } + + /// Execution ended (physically). + pub(crate) fn finished(&self) { + lock(&self.inner).running = false; + } + + pub(crate) fn is_running(&self) -> bool { + lock(&self.inner).running + } + + /// Emit step 1 (blocking): take an emit credit, waiting while all are in + /// flight. Fails with `JobGone` once the job is cancelled or notified. + pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { + let mut inner = lock(&self.inner); + loop { + if !inner.accepting() { + return Err(JobGone); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return Ok(()); + } + // A blocking job runs inline on the simulation thread (D2): the + // credit it waits for comes back only when the owner dispatches + // an emit, which cannot happen while this poll blocks. + #[cfg(all(spawned_sim, feature = "async"))] + if spawned_rt::tasks::Handle::try_current().is_ok() { + drop(inner); + panic!( + "spawned_sim: a blocking job ran out of emit credits inline on the \ + simulation thread (the owner cannot dispatch its emits until the job's \ + step ends); raise JobOptions emit credits or emit less per job" + ); + } + inner = self.emitters.wait(inner).unwrap_or_else(|p| p.into_inner()); + } + } + + /// Emit step 1 (async): take a credit, or call `register` (store the + /// waker) under the job's lock and return `None`. Registration under the + /// lock orders it against the credit return and the cancel, which wake + /// the registered wakers after their update. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn try_acquire(&self, register: impl FnOnce()) -> Option> { + let mut inner = lock(&self.inner); + if !inner.accepting() { + return Some(Err(JobGone)); + } + if inner.emit_credits > 0 { + inner.emit_credits -= 1; + return Some(Ok(())); + } + register(); + None + } + + /// Emit step 2: under the job's lock, enqueue unless the job is + /// cancelled or notified. The caller's credit guard is the only owner of + /// the credit's return: it travels inside `enqueue` and returns the + /// credit when that closure is dropped uncalled (refused here) or when + /// an envelope the mailbox refused is dropped after the lock is released. + pub(crate) fn enqueue_emit( + &self, + enqueue: impl FnOnce() -> Result<(), X>, + ) -> Result<(), (JobGone, Option)> { + let inner = lock(&self.inner); + if !inner.accepting() { + drop(inner); + // `enqueue` (and the credit guard it owns) drops here, unlocked. + drop(enqueue); + return Err((JobGone, None)); + } + enqueue().map_err(|unsent| (JobGone, Some(unsent))) + } + + /// Delayed emit, step 2 at the call: with a credit taken, register a + /// scheduled emit unless the job is cancelled, notified or finished. + /// The caller's credit guard returns the credit on failure. + pub(crate) fn schedule(&self) -> Result<(), JobGone> { + let mut inner = lock(&self.inner); + if !inner.accepting() { + return Err(JobGone); + } + inner.scheduled += 1; + Ok(()) + } + + /// Delayed emit, step 2 at its deadline, or when dropped unfired + /// (`enqueue` is `None`): under the job's lock, enqueue if the + /// notification is still pending. A message not enqueued (the refused + /// envelope, or the uncalled `enqueue` handed back as the third value) + /// returns its credit through its guard when the caller drops it, after + /// the lock: it is user payload. The last scheduled emit hands back a + /// held terminal, to run outside the lock. + pub(crate) fn fire Result<(), X>>( + &self, + enqueue: Option, + ) -> (EmitResult, Option, Option) { + let mut inner = lock(&self.inner); + debug_assert!(inner.scheduled > 0, "scheduled emit underflow"); + inner.scheduled = inner.scheduled.saturating_sub(1); + let (result, unused) = match enqueue { + Some(enqueue) if inner.open() => { + (enqueue().map_err(|unsent| (JobGone, Some(unsent))), None) + } + unused => (Err((JobGone, None)), unused), + }; + let held = if inner.scheduled == 0 { + inner.held.take() + } else { + None + }; + drop(inner); + self.emitters.notify_all(); + (result, held, unused) + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn scheduled(&self) -> u16 { + lock(&self.inner).scheduled + } + + /// Emit step 3: a credit comes back when its message is dispatched, + /// fenced or dropped with the mailbox. The caller wakes async emitters. + pub(crate) fn return_emit_credit(&self) { + lock(&self.inner).emit_credits += 1; + self.emitters.notify_all(); + } +} + +/// What the job set needs from a job. +pub(crate) trait Job { + fn cancel(&self); + /// The owner's exit cancels the job: like [`cancel`](Self::cancel), but + /// the payloads it releases are kept for the owner to destroy after its + /// death point. + fn cancel_for_exit(&self) { + self.cancel(); + } + fn is_running(&self) -> bool; + fn finished(&self); + /// Kept in the job list at compaction (so `cancel_all` reaches it): + /// running, or with work left such as scheduled emits. + fn is_live(&self) -> bool { + self.is_running() + } +} + +/// Compact the weak job list when it reaches this length (and every power of +/// two above it). +const COMPACT_AT: usize = 32; + +/// The job list and the admission gate, under one lock: admission checks +/// the gate and lists the job in one step, so an admitted job is always in +/// the list the owner's exit takes and cancels. +struct JobList { + jobs: Vec>, + /// The owner's exit closed admission (`cancel_all`). + closed: bool, +} + +/// An owner instance's jobs: key map, running count, native budget. +pub(crate) struct JobSetCore { + jobs: Mutex>, + keys: Mutex>>, + running: Mutex, + idle: Condvar, + native: AtomicUsize, + max_native: usize, +} + +impl JobSetCore { + pub(crate) fn new(max_native: usize) -> Self { + Self { + jobs: Mutex::new(JobList { + jobs: Vec::new(), + closed: false, + }), + keys: Mutex::new(HashMap::new()), + running: Mutex::new(0), + idle: Condvar::new(), + native: AtomicUsize::new(0), + max_native, + } + } + + /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. + pub(crate) fn reserve_native(&self) -> bool { + self.native + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| { + (n < self.max_native).then_some(n + 1) + }) + .is_ok() + } + + pub(crate) fn release_native(&self) { + let previous = self.native.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "native job budget underflow"); + } + + pub(crate) fn max_native(&self) -> usize { + self.max_native + } + + #[allow(dead_code)] // used by the Loom model + pub(crate) fn native(&self) -> usize { + self.native.load(Ordering::Acquire) + } + + /// Admit a job: refused (`false`) once the owner's exit closed + /// admission. Otherwise list it, count it as running and replace + /// (cancel) the previous job with the same key, whatever its + /// notification state. The check, the listing and the count are one + /// step under the list's lock, the same lock `cancel_all` closes and + /// drains under, so no admitted job escapes the owner's cancellation and + /// the owner's join counts it. The replaced job and the references + /// compaction upgraded are cancelled and dropped after the locks are + /// released. + #[allow(dead_code)] // used by the Loom model; the job set orders its own + pub(crate) fn admit(&self, state: &Arc, key: Option) -> bool { + match self.admit_replacing(state, key) { + Some(previous) => { + if let Some(previous) = &previous { + previous.cancel(); + } + true + } + None => false, + } + } + + /// [`admit`](Self::admit) that hands the job its key replaced back + /// uncancelled (`Some(None)` when there was none; `None` when refused), + /// so the caller cancels it in the right order and drops its payloads + /// outside its own critical sections. + pub(crate) fn admit_replacing(&self, state: &Arc, key: Option) -> Option>> { + let mut upgraded: Vec> = Vec::new(); + let previous = { + let mut list = lock(&self.jobs); + if list.closed { + return None; + } + let previous = key.and_then(|key| lock(&self.keys).insert(key, state.clone())); + let jobs = &mut list.jobs; + if jobs.len() >= COMPACT_AT && jobs.len().is_power_of_two() { + jobs.retain(|job| match job.upgrade() { + Some(job) => { + let keep = job.is_live(); + upgraded.push(job); + keep + } + None => false, + }); + } + jobs.push(Arc::downgrade(state)); + *lock(&self.running) += 1; + previous + }; + drop(upgraded); + Some(previous) + } + + /// A job's execution ended. Returns whether the set became idle (the + /// caller wakes async joiners). + pub(crate) fn ended(&self, state: &J) -> bool { + state.finished(); + // The key keeps its finished job until replaced: a replacement must + // still cancel it, so its queued terminal message and emits are + // fenced (`NoStaleKeyed`). The map holds one job per key. + let idle = { + let mut running = lock(&self.running); + *running -= 1; + *running == 0 + }; + if idle { + self.idle.notify_all(); + } + idle + } + + pub(crate) fn running(&self) -> usize { + *lock(&self.running) + } + + /// Owner exit: close admission and cancel every job (flag set, blocked + /// emitters woken; [`Job::cancel_for_exit`]). Returns the cancelled jobs + /// (the caller drops them, outside the locks); the key map's references + /// drop outside the locks. + pub(crate) fn cancel_all(&self) -> Vec> { + let (jobs, keys) = { + let mut list = lock(&self.jobs); + list.closed = true; + ( + std::mem::take(&mut list.jobs), + std::mem::take(&mut *lock(&self.keys)), + ) + }; + let cancelled: Vec> = jobs.iter().filter_map(Weak::upgrade).collect(); + for job in &cancelled { + job.cancel_for_exit(); + } + drop(keys); + cancelled + } + + /// Wait until no job is running. `wait` parks on the idle condition + /// variable and returns `Err` with the guard to give up (the cleanup + /// deadline passed). Returns how many jobs are still running. + pub(crate) fn join_with<'a>( + &'a self, + mut wait: impl FnMut( + &'a Condvar, + MutexGuard<'a, usize>, + ) -> Result, MutexGuard<'a, usize>>, + ) -> usize { + let mut running = lock(&self.running); + while *running > 0 { + match wait(&self.idle, running) { + Ok(guard) => running = guard, + Err(guard) => return *guard, + } + } + 0 + } + + /// Take and release the set's locks: the Loom model calls it from a + /// job's final destruction, which deadlocks if that destruction runs + /// under one of them. + #[allow(dead_code)] // used by the Loom model + pub(crate) fn probe_locks(&self) { + drop(lock(&self.jobs)); + drop(lock(&self.keys)); + drop(lock(&self.running)); + } +} diff --git a/concurrency/src/reply/jobs.rs b/concurrency/src/reply/jobs.rs new file mode 100644 index 000000000..f99ca3803 --- /dev/null +++ b/concurrency/src/reply/jobs.rs @@ -0,0 +1,1239 @@ +//! Pipe-to-self jobs: the mode-independent core +//! (docs/design/deferred-replies.md ยง8, ยง9). +//! +//! Each job has one shared `JobState` with three parts: +//! +//! - **execution** (running or finished), physical: cancelled or timed-out +//! work may keep running, and keeps its credit (or native slot) until it +//! ends; +//! - **cancellation**, a flag set by `JobHandle::cancel`, a newer job with the +//! same key, owner exit, or (opt-in) the linked caller leaving; it wakes +//! blocked emitters and suppresses later results; +//! - **terminal notification** (pending, notified, suppressed): a one-shot +//! under the job's lock. The result, a caught panic, or the job deadline +//! takes it and enqueues the terminal message; emits check the same lock, +//! so no emit follows the terminal message. +//! +//! Messages a job produces carry its `JobState` and the owner's `ActorId` and +//! are fenced at dispatch when the job is cancelled. + +use super::deadline::Expirable; +use super::registry::Registry; +use super::slot::CreditPool; +use crate::ActorId; +use std::sync::{Arc, Condvar, Mutex, MutexGuard}; + +/// Default emit credits per job (`JobOptions::emit_credits`). +pub(crate) const DEFAULT_EMIT_CREDITS: u16 = 16; + +/// Default `Actor::MAX_NATIVE_JOBS`: `min(4, available_parallelism)`. +pub(crate) fn default_max_native_jobs() -> usize { + std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1) + .min(4) +} + +/// Per-instance overrides of `Actor::MAX_OUTSTANDING` and +/// `Actor::MAX_NATIVE_JOBS` (set per `ChildSpec`); `None` keeps the actor's +/// constant. +#[cfg(feature = "async")] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Bounds { + pub(crate) max_outstanding: Option, + pub(crate) max_native_jobs: Option, +} + +#[cfg(feature = "async")] +impl Bounds { + /// The effective `(max_outstanding, max_native_jobs)` for an actor whose + /// constants are `outstanding` and `native`. + pub(crate) fn resolve(self, outstanding: usize, native: Option) -> (usize, usize) { + ( + self.max_outstanding.unwrap_or(outstanding), + self.max_native_jobs + .or(native) + .unwrap_or_else(default_max_native_jobs), + ) + } +} + +/// A key for cancel-previous jobs (`JobOptions::key`). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct JobKey(std::borrow::Cow<'static, str>); + +impl JobKey { + pub const fn new(name: &'static str) -> Self { + Self(std::borrow::Cow::Borrowed(name)) + } + pub fn owned(name: String) -> Self { + Self(std::borrow::Cow::Owned(name)) + } +} +impl From<&'static str> for JobKey { + fn from(name: &'static str) -> Self { + Self::new(name) + } +} + +/// Options of a job, passed first to every `pipe*` call and to +/// `reserve_*_job` (as OTP's `*_opt` functions take an option list). +/// Non-exhaustive: build it with the methods, starting from +/// [`JobOptions::new`] (or `default()`). +/// +/// ```ignore +/// ctx.pipe(JobOptions::new().key("fetch").deadline(Duration::from_secs(2)), work, done)?; +/// ``` +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct JobOptions { + pub(crate) deadline: Option, + pub(crate) key: Option, + pub(crate) cancel_on_caller_gone: bool, + pub(crate) emit_credits: u16, +} + +impl JobOptions { + /// No deadline, no key, 16 emit credits, not cancelled when the caller + /// leaves. + pub fn new() -> Self { + Self::default() + } + + /// Cancel the previous job with this key (whatever its notification + /// state) and fence its queued messages: the replacement is the only + /// live job of the key. + pub fn key(mut self, key: impl Into) -> Self { + self.key = Some(key.into()); + self + } + + /// A hard deadline: a `Duration` from now or an `Instant` (on the + /// runtime's clock, the simulated one under `spawned_sim`), or + /// `Deadline::Never`. When it passes, `done` receives + /// `Err(JobError::DeadlineExceeded)`, the job's later emits fail with + /// [`JobGone`] and its eventual result is dropped; the work keeps + /// running (cooperative cancellation) and keeps its credit. For a soft + /// deadline (stop starting new work, finish what is in hand), check the + /// time inside the work instead. + pub fn deadline(mut self, deadline: impl Into) -> Self { + self.deadline = match deadline.into() { + super::Deadline::At(at) => Some(at), + super::Deadline::Never => None, + }; + self + } + + /// Emits in flight before `emit` waits (`emit_blocking` blocks). The + /// default is 16. + pub fn emit_credits(mut self, credits: std::num::NonZeroU16) -> Self { + self.emit_credits = credits.get(); + self + } + + /// Reply jobs (`pipe_reply*`) only: cancel the job when its caller + /// leaves or the request's deadline passes. Off by default, as in OTP. + /// Plain `pipe*` jobs have no caller: setting it there is a bug + /// (`debug_assert!`). + pub fn cancel_on_caller_gone(mut self) -> Self { + self.cancel_on_caller_gone = true; + self + } +} + +impl Default for JobOptions { + fn default() -> Self { + Self { + deadline: None, + key: None, + cancel_on_caller_gone: false, + emit_credits: DEFAULT_EMIT_CREDITS, + } + } +} + +/// How a job ended, delivered to `done` in the owner's turn: the work's +/// value, or why there is none. A plain `Result`, so `?` and every +/// `Result` combinator work. +pub type JobOutcome = Result; + +/// Why a job produced no value. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum JobError { + /// The work panicked; caught at the job boundary. + #[error("job panicked: {0}")] + Panicked(String), + /// The job's deadline (`JobOptions::deadline`) passed first. The + /// deadline is hard: the job's later emits fail and its eventual result + /// is dropped. The work may still be running (cooperative + /// cancellation). + #[error("job deadline exceeded")] + DeadlineExceeded, +} + +/// Why a job was not spawned. No job runs, and any credit or native slot +/// reserved for it was released. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum PipeError { + /// The owner's `MAX_OUTSTANDING` credits are in use. + #[error("deferred credits at the bound")] + Overloaded, + /// The owner's `MAX_NATIVE_JOBS` native jobs are running. + #[error("native job budget at the bound")] + NativeBudget, + /// The native thread could not be spawned. + #[error("spawning the job failed: {0}")] + SpawnFailed(std::io::Error), + /// The owner is exiting. + #[error("the owner is stopping")] + OwnerStopping, +} + +/// `pipe_reply*` failed; the token comes back in every case. +#[derive(Debug)] +#[non_exhaustive] +pub enum PipeReplyError { + /// The job could not be spawned. + Failed { + token: super::ReplyTo, + error: PipeError, + }, + /// The request is no longer open (or past its deadline). + Closed(super::ReplyTo), + /// `token.owner()` is not this actor. + WrongOwner(super::ReplyTo), +} + +impl PipeReplyError { + /// The token handed back. + pub fn into_token(self) -> super::ReplyTo { + match self { + Self::Failed { token, .. } | Self::Closed(token) | Self::WrongOwner(token) => token, + } + } +} + +pub use super::job_core::JobGone; +use super::job_core::{Finish, HeldTerminal, Job, JobCore, JobSetCore}; +use crate::lifecycle_trace as trace; + +/// One job's shared state: the core protocol plus the async wake-ups. +pub(crate) struct JobState { + owner: ActorId, + core: JobCore, + /// The job's number in lifecycle traces. + #[cfg(feature = "lifecycle-trace")] + trace: u64, + /// A `pipe_reply*` job: its request's slot in reply traces, and whether + /// its notification was taken (DeferredReplies.tla `JobNotify`, `JobEnds`, + /// `JobCancel`). + #[cfg(feature = "lifecycle-trace")] + link: std::sync::OnceLock, + #[cfg(feature = "lifecycle-trace")] + notified: std::sync::atomic::AtomicBool, + /// Async cancellation for `tasks` jobs (drops the work future). + #[cfg(feature = "async")] + cancel: spawned_rt::tasks::CancellationToken, + /// Wakers of async emitters waiting for a credit. + #[cfg(feature = "async")] + emit_wakers: Mutex>, + /// Delayed emits (`JobCtx::emit_at`) not yet fired: their strong + /// references (the deadline heap holds `Weak`s). Cleared at cancellation. + scheduled: Mutex>>, + /// What the owner's exit released (a held result, scheduled emits): + /// user payload, destroyed by [`JobSet::dispose_cancelled`] once the + /// owner's death point answered and woke every caller. + exit_released: Mutex>, +} + +/// Payloads an owner's exit took from a job, destroyed after its death point. +type ExitReleased = (Option, Vec>); + +fn lock(mutex: &Mutex) -> MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +impl JobState { + pub(crate) fn new(owner: ActorId, emit_credits: u16) -> Arc { + Arc::new(Self { + owner, + core: JobCore::new(emit_credits), + #[cfg(feature = "lifecycle-trace")] + trace: crate::lifecycle_trace::next_job(), + #[cfg(feature = "lifecycle-trace")] + link: std::sync::OnceLock::new(), + #[cfg(feature = "lifecycle-trace")] + notified: std::sync::atomic::AtomicBool::new(false), + #[cfg(feature = "async")] + cancel: spawned_rt::tasks::CancellationToken::new(), + #[cfg(feature = "async")] + emit_wakers: Mutex::new(Vec::new()), + scheduled: Mutex::new(Vec::new()), + exit_released: Mutex::new(None), + }) + } + + pub(crate) fn is_cancelled(&self) -> bool { + self.core.is_cancelled() + } + + #[cfg(feature = "async")] + pub(crate) fn cancellation(&self) -> &spawned_rt::tasks::CancellationToken { + &self.cancel + } + + /// A `pipe_reply*` job was launched for the request in `slot` (reply + /// traces: the model's `PipeReply`). + #[cfg(feature = "lifecycle-trace")] + pub(crate) fn linked(&self, slot: u64) { + trace::atomic(|| { + let _ = self.link.set(slot); + trace::reply(self.owner.raw(), "PipeReply", slot, "linked"); + }); + } + + // DeferredReplies.tla linked-job actions of a `pipe_reply*` job, recorded + // on its owner's reply trace inside the trace section. + + /// Its notification was taken: the token travels with it from now on. + fn link_notified(&self) { + #[cfg(feature = "lifecycle-trace")] + self.notified + .store(true, std::sync::atomic::Ordering::Relaxed); + } + + /// `JobNotify`: its done or deadline message is dispatched to the owner. + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn link_notify(&self, kind: &'static str) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + trace::reply(self.owner.raw(), "JobNotify", *slot, kind); + } + } + + /// `JobEnds`: execution ended without taking the notification. + fn link_end(&self) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + trace::reply(self.owner.raw(), "JobEnd", *slot, ""); + } + } + + /// `JobCancel`: cancelled while the token is still with the job (not + /// once its notification was taken). + fn link_cancel(&self) { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = self.link.get() { + if !self.notified.load(std::sync::atomic::Ordering::Relaxed) { + trace::reply(self.owner.raw(), "JobCancel", *slot, ""); + } + } + } + + /// Record a `DeferredJobs.tla` action of this job (lifecycle traces). + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + fn record(&self, event: impl FnOnce(u64) -> trace::JobEvent) { + #[cfg(feature = "lifecycle-trace")] + trace::job(self.owner.raw(), || event(self.trace)); + } + + /// Set the cancellation flag: suppresses a pending notification and + /// wakes blocked emitters. Never undoes effects. Recorded as the model's + /// `Cancel` (the owner's exit and key replacement cancel through the job + /// set instead, as part of `Exit` and `Start`). + pub(crate) fn cancel(&self) { + let held = trace::atomic(|| { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { + self.count_cancel(); + self.record(|job| trace::JobEvent::new("Cancel", job)); + self.link_cancel(); + } + held + }); + // The suppressed held result (user payload): outside the trace + // section, after its Cancel and JobCancel are recorded. + drop(held); + self.wake_cancelled(); + } + + /// A permit used after the owner began exiting: the job never runs. + pub(crate) fn never_started(&self) { + self.core.cancel(); + self.core.finished(); + } + + /// A keyed replacement cancels this job (inside the trace section, + /// after the new job's `Start`): the flag, its count and a linked + /// caller's `JobCancel`; the suppressed held result comes back to be + /// dropped outside, with [`wake_cancelled`](Self::wake_cancelled). + fn cancel_replaced(&self) -> Option { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { + self.count_cancel(); + self.link_cancel(); + } + held + } + + fn cancel_quietly(&self) { + if self.core.cancel() { + self.count_cancel(); + } + self.wake_cancelled(); + } + + /// `spawned_jobs_cancelled_total` counts jobs cancelled while running. + fn count_cancel(&self) { + if self.core.is_running() { + crate::instrumentation::hooks::job_cancelled(); + } + } + + fn wake_cancelled(&self) { + // Scheduled emits will never be delivered: drop them now (their + // messages and credits), outside every lock. + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + for emit in scheduled { + emit.discard(self); + } + #[cfg(feature = "async")] + { + self.cancel.cancel(); + self.wake_emitters(); + } + } + + #[cfg(feature = "async")] + fn wake_emitters(&self) { + let wakers = std::mem::take(&mut *lock(&self.emit_wakers)); + for waker in wakers { + waker.wake(); + } + } + + /// Take the terminal notification and run `enqueue` under the job's + /// lock (see [`JobCore::notify`]). `end` is `ok`, `panicked` (the end + /// of execution, the model's `Finish`) or `deadline` (`JobDeadline`). + pub(crate) fn notify( + &self, + end: &'static str, + enqueue: impl FnOnce() -> Option, + ) -> (bool, Option) { + let deadline = end == "deadline"; + let (taken, unsent, held) = trace::atomic(|| { + let (taken, unsent, held) = self.core.notify_keeping_held(enqueue); + if taken { + crate::instrumentation::hooks::job_notified(end); + self.link_notified(); + } else if !deadline { + self.link_end(); + } + if !deadline { + // Only a held result's terminal notifies without the + // deadline (the work's own result goes through `finish`): + // the model's ReleaseHeld. + self.record(|job| trace::JobEvent { + x: if taken { "notified" } else { "quiet" }, + ..trace::JobEvent::new("Release", job) + }); + } else if taken { + self.record(|job| trace::JobEvent::new("JobDeadline", job)); + } + (taken, unsent, held) + }); + // A held result the deadline displaced (user payload): dropped + // outside the trace section, after JobDeadline is recorded. + drop(held); + #[cfg(feature = "async")] + if taken { + self.wake_emitters(); + } + (taken, unsent) + } + + /// The work's result (see [`JobCore::finish`]): taken now, or held + /// until the job's scheduled emits have fired (`later` then runs + /// `notify`). Returns the refused envelope, to drop outside locks. + pub(crate) fn finish( + &self, + end: &'static str, + enqueue: impl FnOnce() -> Option, + later: impl FnOnce() -> HeldTerminal, + ) -> Option { + let mut enqueue = Some(enqueue); + let (finish, unsent) = trace::atomic(|| { + let (finish, unsent) = self + .core + .finish(|| enqueue.take().and_then(|enqueue| enqueue()), later); + match finish { + Finish::Taken => { + crate::instrumentation::hooks::job_notified(end); + self.link_notified(); + self.record(|job| trace::JobEvent { + x: "notified", + ..trace::JobEvent::new("Finish", job) + }); + } + Finish::Refused => { + self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }); + self.link_end(); + } + Finish::Held => self.record(|job| trace::JobEvent { + x: "held", + ..trace::JobEvent::new("Finish", job) + }), + } + (finish, unsent) + }); + #[cfg(feature = "async")] + if finish == Finish::Taken { + self.wake_emitters(); + } + #[cfg(not(feature = "async"))] + let _ = finish; + unsent + } + + /// Register a delayed emit at `at` (ticks): `send` enqueues its envelope + /// when the deadline fires, fenced like any emit, and the job's terminal + /// message waits for it. `track` arms the owner's deadline heap. On + /// `Err` the caller's credit guard (inside `send`) returns the credit. + /// + /// `JobGone` also when the emit will never be delivered: the job was + /// cancelled while registering, or an emit already due was refused. An + /// `at` past the job's deadline is accepted: the deadline takes the + /// notification first and the emit is dropped at its instant. + pub(crate) fn schedule_emit( + self: &Arc, + at: u64, + send: SendScheduled, + failures: Arc, + track: impl FnOnce(u64, std::sync::Weak), + ) -> Result<(), JobGone> { + trace::atomic(|| { + let scheduled = self.core.schedule(); + if scheduled.is_ok() { + self.record(|job| trace::JobEvent::new("Schedule", job)); + } + scheduled + })?; + #[cfg(test)] + tests::after_schedule(); + let emit = Arc::new(ScheduledEmit { + at, + job: Arc::downgrade(self), + send: Mutex::new(Some(send)), + failures, + }); + lock(&self.scheduled).push(emit.clone()); + if self.is_cancelled() { + // Cancelled between the check and the registration: the cancel + // may have missed it. Take the list in its own statement: the + // discard locks it again (`settle`), so its guard must be gone. + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + for emit in scheduled { + emit.discard(self); + } + return Err(JobGone); + } + if at <= super::now_ticks() { + if !emit.settle(self, true) { + return Err(JobGone); + } + } else { + let weak: std::sync::Weak = Arc::downgrade(&emit) as _; + track(at, weak); + } + Ok(()) + } + + pub(crate) fn is_running(&self) -> bool { + self.core.is_running() + } + + /// Free emit credits (tests). + #[cfg(test)] + pub(crate) fn emit_credits(&self) -> u16 { + self.core.emit_credits() + } + + /// Execution of a cancelled async job ended without a notification (the + /// model's `Finish` when the notification was suppressed). + #[cfg(feature = "async")] + pub(crate) fn ended_cancelled(&self) { + trace::atomic(|| { + self.record(|job| trace::JobEvent { + x: "quiet", + ..trace::JobEvent::new("Finish", job) + }); + self.link_end(); + }); + } + + /// The dispatch fence: a message of this job dispatched by instance `at` + /// is dropped when the job is cancelled or belongs to another instance. + /// `kind` is `emit`, `done` or `deadline` (the model's `Dispatch`). + pub(crate) fn fenced(&self, at: ActorId, kind: &'static str) -> bool { + trace::atomic(|| { + let fenced = self.is_cancelled() || self.owner != at; + if fenced { + crate::instrumentation::hooks::job_fenced(); + } + self.record(|job| trace::JobEvent { + x: kind, + fenced, + ..trace::JobEvent::new("Dispatch", job) + }); + if !fenced && matches!(kind, "done" | "deadline") { + self.link_notify(kind); + } + fenced + }) + } + + /// Emit step 1 (blocking). + pub(crate) fn acquire_blocking(&self) -> Result<(), JobGone> { + self.core.acquire_blocking() + } + + /// Emit step 1 (async). + #[cfg(feature = "async")] + pub(crate) fn poll_acquire( + &self, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + let acquired = self.core.try_acquire(|| { + let mut wakers = lock(&self.emit_wakers); + if !wakers.iter().any(|w| w.will_wake(cx.waker())) { + wakers.push(cx.waker().clone()); + } + }); + match acquired { + Some(result) => std::task::Poll::Ready(result), + None => std::task::Poll::Pending, + } + } + + /// Emit step 2 (see [`JobCore::enqueue_emit`]). + pub(crate) fn enqueue_emit( + &self, + enqueue: impl FnOnce() -> Result<(), X>, + ) -> Result<(), (JobGone, Option)> { + trace::atomic(|| { + let result = self.core.enqueue_emit(enqueue); + if result.is_ok() { + crate::instrumentation::hooks::job_emitted(); + self.record(|job| trace::JobEvent::new("Emit", job)); + } + result + }) + } + + /// Emit step 3: a credit comes back when its message is dispatched, + /// fenced or dropped with the mailbox. + pub(crate) fn return_emit_credit(&self) { + self.core.return_emit_credit(); + #[cfg(feature = "async")] + self.wake_emitters(); + } +} + +impl Job for JobState { + fn cancel(&self) { + self.cancel_quietly(); + } + /// Mark the job cancelled (fences its messages, wakes its emitters, + /// stops its async work) without destroying anything: the suppressed + /// held result and the scheduled emits are kept for + /// [`JobSet::dispose_cancelled`]. + fn cancel_for_exit(&self) { + let (newly, held) = self.core.cancel_keeping_held(); + if newly { + self.count_cancel(); + self.link_cancel(); + } + let scheduled = std::mem::take(&mut *lock(&self.scheduled)); + *lock(&self.exit_released) = Some((held, scheduled)); + // The cancellation flag is set (`is_cancelled`, `emit` refuse from + // here); the callbacks it owes (the `cancelled()` waiters' wakers + // and blocked emitters, arbitrary code) run in `dispose_cancelled`, + // after the owner's callers are answered. + } + fn is_running(&self) -> bool { + self.core.is_running() + } + fn is_live(&self) -> bool { + // Scheduled emits keep a finished job reachable for `cancel_all`. + self.core.is_running() || self.core.scheduled() > 0 + } + fn finished(&self) { + self.core.finished(); + } +} + +/// A handle to a spawned job. Dropping it leaves the job owned by the +/// actor's job set (it is not detached). +#[derive(Clone)] +pub struct JobHandle { + state: Arc, +} + +impl std::fmt::Debug for JobHandle { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobHandle") + .field("owner", &self.state.owner) + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobHandle { + pub(crate) fn new(state: Arc) -> Self { + Self { state } + } + /// Cancel the job: its later results and queued messages are fenced, and + /// blocked emitters return `JobGone`. Work already done is not undone. + pub fn cancel(&self) { + self.state.cancel(); + } + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + /// Whether the job's work is still executing (cancelled work may still + /// run until it returns). + pub fn is_running(&self) -> bool { + self.state.is_running() + } +} + +// --------------------------------------------------------------------------- +// Job set +// --------------------------------------------------------------------------- + +/// An owner instance's jobs: key map, running count, native budget. +pub(crate) struct JobSet { + owner: ActorId, + core: JobSetCore, + /// Whether the owner's exit was recorded (lifecycle traces). + #[cfg(feature = "lifecycle-trace")] + exited: std::sync::atomic::AtomicBool, + /// Jobs the owner's exit cancelled, whose released payloads wait for + /// [`dispose_cancelled`](Self::dispose_cancelled). + exit_cancelled: Mutex>>, + /// The first `NativeBudget` refusal was logged (only refusals touch it). + native_refusal_logged: std::sync::atomic::AtomicBool, + #[cfg(feature = "async")] + idle_async: Mutex>, +} + +impl JobSet { + pub(crate) fn new(owner: ActorId, max_native: usize) -> Self { + Self { + owner, + core: JobSetCore::new(max_native), + #[cfg(feature = "lifecycle-trace")] + exited: std::sync::atomic::AtomicBool::new(false), + exit_cancelled: Mutex::new(Vec::new()), + native_refusal_logged: std::sync::atomic::AtomicBool::new(false), + #[cfg(feature = "async")] + idle_async: Mutex::new(Vec::new()), + } + } + + pub(crate) fn owner(&self) -> ActorId { + self.owner + } + + /// Reserve a native slot (`MAX_NATIVE_JOBS`), rejecting at the bound. + /// The first refusal per instance is logged (a `warn`): under load the + /// default budget is easy to miss, and `NativeBudget` is otherwise the + /// only symptom. The reserving path itself is unchanged. + pub(crate) fn reserve_native(&self) -> bool { + let reserved = self.core.reserve_native(); + if !reserved { + crate::instrumentation::hooks::job_rejected(); + if !self + .native_refusal_logged + .swap(true, std::sync::atomic::Ordering::Relaxed) + { + let max_native_jobs = self.core.max_native(); + crate::actor_cleanup::log_safely(|| { + tracing::warn!(event = "native_budget_exhausted", actor = %self.owner, + max_native_jobs, "blocking job refused (NativeBudget): all native \ + job slots are in use; raise MAX_NATIVE_JOBS \ + (#[actor(max_native_jobs = N)] or ChildSpec::max_native_jobs) if \ + this is expected load. Logged once per actor instance"); + }); + } + } + reserved + } + + pub(crate) fn release_native(&self) { + self.core.release_native(); + } + + /// Admit a job: replace (cancel) the previous one with the same key and + /// count it as running (the model's `Start`). Refused (`false`, nothing + /// recorded) once the owner's exit closed admission: the caller must not + /// run it. + #[must_use] + pub(crate) fn admit(&self, state: &Arc, key: Option, deadline: bool) -> bool { + let admitted = trace::atomic(|| { + let keyed = key.is_some(); + let previous = self.core.admit_replacing(state, key)?; + crate::instrumentation::hooks::job_started(); + state.record(|job| trace::JobEvent { + keyed, + deadline, + ..trace::JobEvent::new("Start", job) + }); + // The replaced job is cancelled in the same step as the new + // job's Start (the model's CancelCore(ToReplace)); a linked + // job's caller sees its JobCancel. Nothing is destroyed here. + Some(previous.map(|previous| { + let held = previous.cancel_replaced(); + (previous, held) + })) + }); + let Some(replaced) = admitted else { + return false; + }; + // Outside the trace section and every lock: the replaced job's + // suppressed result and scheduled emits (user payloads). + if let Some((previous, held)) = replaced { + drop(held); + previous.wake_cancelled(); + } + true + } + + /// A job's execution ended. + pub(crate) fn ended(&self, state: &Arc) { + crate::instrumentation::hooks::job_ended(); + if self.core.ended(state) { + #[cfg(feature = "async")] + for waker in std::mem::take(&mut *lock(&self.idle_async)) { + waker.wake(); + } + } + } + + pub(crate) fn running(&self) -> usize { + self.core.running() + } + + /// Owner exit: close admission and cancel every job (flag set, blocked + /// emitters woken). Nothing is destroyed here: what the jobs release + /// waits for [`dispose_cancelled`](Self::dispose_cancelled). + pub(crate) fn cancel_all(&self) { + trace::atomic(|| { + let cancelled = self.core.cancel_all(); + lock(&self.exit_cancelled).extend(cancelled); + #[cfg(feature = "lifecycle-trace")] + if !self.exited.swap(true, std::sync::atomic::Ordering::AcqRel) { + trace::job(self.owner.raw(), || trace::JobEvent::new("Exit", 0)); + } + }); + } + + /// After the owner's death point answered and woke every caller: run + /// the callbacks the exit's cancellation owes (wake `cancelled()` + /// waiters and blocked emitters), then destroy what it released (held + /// results, scheduled emits; contained through each job's failure + /// sink). Idempotent. + pub(crate) fn dispose_cancelled(&self) { + let cancelled = std::mem::take(&mut *lock(&self.exit_cancelled)); + #[cfg(feature = "async")] + for job in &cancelled { + job.cancel.cancel(); + job.wake_emitters(); + } + for job in &cancelled { + let released = lock(&job.exit_released).take(); + if let Some((held, scheduled)) = released { + drop(held); + for emit in scheduled { + emit.discard(job); + } + } + } + drop(cancelled); + } + + /// Join the cancelled jobs, bounded by `deadline`; returns how many are + /// still running (unfinished at stop). + pub(crate) fn join_blocking(&self, deadline: std::time::Instant) -> usize { + self.core.join_with(|idle, running| { + let now = std::time::Instant::now(); + if now >= deadline { + return Err(running); + } + Ok(idle + .wait_timeout(running, deadline - now) + .unwrap_or_else(|p| p.into_inner()) + .0) + }) + } + + /// Async join, bounded by `timeout` on the runtime's clock. + #[cfg(feature = "async")] + pub(crate) async fn join(&self, timeout: std::time::Duration) -> usize { + let idle = std::future::poll_fn(|cx| { + if self.running() == 0 { + return std::task::Poll::Ready(()); + } + lock(&self.idle_async).push(cx.waker().clone()); + if self.running() == 0 { + return std::task::Poll::Ready(()); + } + std::task::Poll::Pending + }); + let _ = spawned_rt::tasks::timeout(timeout, idle).await; + self.running() + } +} + +/// Report jobs still running when the owner's cleanup deadline passed. +pub(crate) fn report_unfinished(owner: ActorId, unfinished: usize) { + if unfinished > 0 { + crate::instrumentation::hooks::jobs_unfinished(unfinished); + tracing::warn!( + event = "unfinished_at_stop", + actor = %owner, + unfinished, + "job cleanup incomplete: jobs still running at the cleanup deadline" + ); + } +} + +// --------------------------------------------------------------------------- +// Credits and deadlines +// --------------------------------------------------------------------------- + +/// The credit an unlinked job holds while it executes. +pub(crate) struct JobCredit(Option>); + +impl JobCredit { + pub(crate) fn reserve(registry: &Registry) -> Option { + let pool = registry.pool().clone(); + // `then`, not `then_some`: the guard must exist only when reserved. + let credit = pool.reserve(1).then(|| Self(Some(pool))); + if credit.is_none() { + crate::instrumentation::hooks::job_rejected(); + } + credit + } +} + +impl Drop for JobCredit { + fn drop(&mut self) { + if let Some(pool) = self.0.take() { + pool.release(1); + } + } +} + +/// The deadline notification of one job, serviced by the owner's heap. +pub(crate) struct JobDeadline { + pub(crate) at: u64, + pub(crate) state: Arc, + pub(crate) notify: Mutex>>, +} + +impl Expirable for JobDeadline { + fn pending_deadline(&self) -> Option { + self.state.core.is_pending().then_some(self.at) + } + fn expire(&self, _now: u64) { + let notify = lock(&self.notify).take(); + if let Some(notify) = notify { + notify(); + } + } +} + +/// A value (user payload) whose drop is contained by the owner's failure +/// sink: it may be dropped on the deadline service thread. +pub(crate) struct Contained { + value: Option, + failures: Arc, +} + +impl Contained { + pub(crate) fn new(value: T, failures: Arc) -> Self { + Self { + value: Some(value), + failures, + } + } + pub(crate) fn into_inner(mut self) -> Option { + self.value.take() + } +} + +impl Drop for Contained { + fn drop(&mut self) { + if let Some(value) = self.value.take() { + self.failures.discard(value); + } + } +} + +/// Enqueues a delayed emit's envelope; a refused envelope comes back erased, +/// to be dropped outside locks. +pub(crate) type SendScheduled = Box Result<(), Box> + Send>; + +/// One delayed emit (`JobCtx::emit_at`), serviced by the owner's deadline +/// heap. Holds its message and emit credit until it fires or is discarded. +/// +/// Its message is user payload and is usually dropped on the process-wide +/// deadline service thread: every drop goes through the owner's failure +/// sink, which contains a panicking destructor (recorded as a cleanup +/// failure), so the emit still settles and the service keeps running. +pub(crate) struct ScheduledEmit { + at: u64, + job: std::sync::Weak, + send: Mutex>, + failures: Arc, +} + +impl Drop for ScheduledEmit { + fn drop(&mut self) { + let send = self + .send + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); + self.failures.discard(send); + } +} + +impl ScheduledEmit { + /// Fire (`deliver`) or discard: step 2 of the emit protocol under the + /// job's lock, then remove it from the job, then run a held terminal. + /// Returns whether the message was enqueued. + fn settle(&self, job: &JobState, deliver: bool) -> bool { + let Some(send) = lock(&self.send).take() else { + return false; + }; + // A discarded message (and its credit guard) drops below, outside the + // trace section and every lock, through the failure sink. + let (deliver, discarded) = if deliver { + (Some(send), None) + } else { + (None, Some(send)) + }; + let (result, held, unused) = trace::atomic(|| { + let (result, held, unused) = job.core.fire(deliver); + if result.is_ok() { + crate::instrumentation::hooks::job_emitted(); + } + // The model's FireScheduled: enqueued or dropped, and whether + // this last fire took a held result (released below). + job.record(|id| trace::JobEvent { + x: if result.is_ok() { "emit" } else { "dropped" }, + release: held.is_some(), + ..trace::JobEvent::new("Fire", id) + }); + (result, held, unused) + }); + let delivered = result.is_ok(); + // Whatever was not enqueued (the refused envelope, the uncalled send + // or the discarded one, each with its credit guard) drops here, + // unlocked and contained. + self.failures.discard((result, unused, discarded)); + lock(&job.scheduled).retain(|emit| !std::ptr::eq(Arc::as_ptr(emit), self)); + if let Some(held) = held { + held(); + } + #[cfg(feature = "async")] + job.wake_emitters(); + delivered + } + + fn discard(&self, job: &JobState) { + self.settle(job, false); + } +} + +impl Expirable for ScheduledEmit { + fn pending_deadline(&self) -> Option { + lock(&self.send).is_some().then_some(self.at) + } + fn expire(&self, _now: u64) { + fail_point!("job.emit_due"); + if let Some(job) = self.job.upgrade() { + self.settle(&job, true); + } + } +} + +/// Threads-mode cleanup completion (death plus the bounded job join), for +/// `join()`. Set when the guard drops, so a panicking cleanup also releases +/// joiners. +#[derive(Default)] +pub(crate) struct Cleaned { + done: Mutex, + changed: Condvar, +} + +impl Cleaned { + pub(crate) fn wait(&self) { + let mut done = lock(&self.done); + while !*done { + done = self.changed.wait(done).unwrap_or_else(|p| p.into_inner()); + } + } +} + +pub(crate) struct CleanedGuard(pub(crate) Arc); + +impl Drop for CleanedGuard { + fn drop(&mut self) { + *lock(&self.0.done) = true; + self.0.changed.notify_all(); + } +} + +/// Opt-in caller-give-up cancellation for jobs that cannot await +/// (`pipe_reply_blocking*` with `cancel_on_caller_gone`): a watcher waker on +/// the linked slot cancels the job when the slot closes. +/// +/// Watchers are also woken when the token's deadline is shortened (so timers +/// re-arm): the waker rechecks the slot and re-registers while it is open, +/// as an awaiting watcher would re-poll. It refers to the slot weakly (the +/// slot stores it), and a slot that is gone has no caller left. +pub(crate) fn cancel_when_closed( + slot: &Arc>, + state: &Arc, +) { + struct CancelJob { + slot: std::sync::Weak>, + state: Arc, + } + impl CancelJob { + /// Register as the slot's watcher; `true` once the slot is closed. + fn closed(self: &Arc) -> bool { + let Some(slot) = self.slot.upgrade() else { + return true; + }; + let waker = std::task::Waker::from(self.clone()); + slot.poll_closed(&waker, &super::now_ticks).is_ready() + } + } + impl std::task::Wake for CancelJob { + fn wake(self: Arc) { + if self.closed() { + self.state.cancel(); + } + } + } + let watcher = Arc::new(CancelJob { + slot: Arc::downgrade(slot), + state: state.clone(), + }); + if watcher.closed() { + state.cancel(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reply::registry::Registry; + + std::thread_local! { + /// Runs once between `schedule_emit`'s `JobCore::schedule` and its + /// registration of the emit (a deterministic interleaving). + static AFTER_SCHEDULE: std::cell::RefCell>> = + const { std::cell::RefCell::new(None) }; + } + + pub(super) fn after_schedule() { + if let Some(hook) = AFTER_SCHEDULE.with(|hook| hook.borrow_mut().take()) { + hook(); + } + } + + /// Review P1 (Codex 3): a cancel that drains the still-empty scheduled + /// list between `JobCore::schedule` and the registration leaves the + /// registration to discard its own emit. It used to iterate under the + /// list's guard while `settle` locked the list again: a self-deadlock. + #[test] + fn cancel_between_schedule_and_registration_discards_without_deadlock() { + let state = JobState::new(ActorId::next(), 2); + let cancelled = state.clone(); + let (sent_tx, sent_rx) = std::sync::mpsc::channel::<()>(); + let (done_tx, done_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + AFTER_SCHEDULE + .with(|hook| *hook.borrow_mut() = Some(Box::new(move || cancelled.cancel()))); + let send: SendScheduled = Box::new(move || { + let _ = sent_tx.send(()); + Ok(()) + }); + let result = state.schedule_emit(u64::MAX - 1, send, Arc::default(), |_, _| {}); + let _ = done_tx.send((result, state.core.scheduled())); + }); + let (result, scheduled) = done_rx + .recv_timeout(std::time::Duration::from_secs(10)) + .expect("schedule_emit deadlocked discarding its own emit"); + assert_eq!(result, Err(JobGone)); + assert_eq!(scheduled, 0, "the discarded emit is no longer scheduled"); + assert!( + sent_rx.try_recv().is_err(), + "a cancelled job's emit was sent" + ); + } + + /// Review P1: a rejected reservation must not build a guard whose drop + /// releases a credit that was never taken. + #[test] + fn a_rejected_credit_releases_nothing() { + let registry = Registry::new(ActorId::next(), 1); + let held = JobCredit::reserve(®istry).expect("first credit"); + assert!(JobCredit::reserve(®istry).is_none()); + assert_eq!( + registry.outstanding(), + 1, + "a rejected job released a credit" + ); + drop(held); + assert_eq!(registry.outstanding(), 0); + } + + /// The first `NativeBudget` refusal per instance is flagged (and + /// logged); a reservation that succeeds never touches the flag. + #[test] + fn the_first_native_refusal_is_flagged_once() { + use std::sync::atomic::Ordering; + let set = JobSet::new(ActorId::next(), 1); + assert!(set.reserve_native()); + assert!(!set.native_refusal_logged.load(Ordering::Relaxed)); + assert!(!set.reserve_native()); + assert!(set.native_refusal_logged.load(Ordering::Relaxed)); + assert!(!set.reserve_native()); + set.release_native(); + } + + /// Review P1: a finished keyed job stays replaceable, so its queued + /// messages are fenced when a newer job takes the key. + #[test] + fn replacement_cancels_a_finished_keyed_job() { + let set = JobSet::new(ActorId::next(), 1); + let old = JobState::new(set.owner(), 1); + assert!(set.admit(&old, Some(JobKey::new("k")), false)); + set.ended(&old); + let new = JobState::new(set.owner(), 1); + assert!(set.admit(&new, Some(JobKey::new("k")), false)); + assert!(old.is_cancelled(), "the finished job escaped replacement"); + assert!(!new.is_cancelled()); + } +} diff --git a/concurrency/src/reply/mod.rs b/concurrency/src/reply/mod.rs new file mode 100644 index 000000000..0fbca77df --- /dev/null +++ b/concurrency/src/reply/mod.rs @@ -0,0 +1,896 @@ +//! Reply transport for requests: reply slots, tokens and receivers +//! (docs/design/deferred-replies.md). +//! +//! Every request has a reply slot (`slot::Slot`) shared by the envelope, +//! then by the handler's [`PendingReply`] (and, after [`PendingReply::defer`], +//! a storable [`ReplyTo`]), and by the caller's [`ReplyReceiver`]. Each slot +//! gets exactly one terminal discharge. + +pub(crate) mod deadline; +pub(crate) mod job_core; +pub mod jobs; +mod pending; +pub(crate) mod registry; +pub(crate) mod slot; +pub(crate) mod turn; + +pub use pending::{InsertError, InsertErrorKind, PendingReplies}; +pub(crate) use registry::DEFAULT_MAX_OUTSTANDING; +pub use slot::Delivery; + +use crate::error::ActorError; +use crate::ActorId; +use registry::Registry; +pub(crate) use slot::Slot; +use slot::{Terminal, NEVER}; +/// `Deadline::Never` in ticks. +pub(crate) const NEVER_TICKS: u64 = NEVER; +use spawned_rt::time::Instant; +use std::borrow::Cow; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; + +/// Production synchronization types for [`slot`]; the Loom model supplies its +/// own (`tests/reply_slot_loom.rs`). +mod sync { + pub(crate) use std::sync::atomic::{AtomicBool, AtomicU64, AtomicU8, AtomicUsize, Ordering}; + pub(crate) use std::sync::{Arc, Arc as JobArc, Condvar, Mutex, MutexGuard, Weak as JobWeak}; + /// Lifecycle-trace builds keep the owner in each slot (its events name + /// it); other builds keep nothing, so a slot stays small. + #[cfg(feature = "lifecycle-trace")] + pub(crate) type SlotOwner = Option>; + #[cfg(not(feature = "lifecycle-trace"))] + pub(crate) type SlotOwner = (); + /// The owner's final reason, as the slot records it at its close. + pub(crate) type Reason = crate::error::ExitReason; + + /// Run a slot transition as one step of the lifecycle-trace + /// linearization (the identity without the feature). + #[inline(always)] + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + crate::lifecycle_trace::atomic(f) + } + + /// A slot left `OPEN` (`kind` is the model's terminal name; `token` + /// whether a deferred token was live): metrics and lifecycle traces. + #[inline(always)] + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + pub(crate) fn left_open(owner: &SlotOwner, slot: usize, kind: &'static str, token: bool) { + crate::instrumentation::hooks::left_open(kind, token); + #[cfg(feature = "lifecycle-trace")] + { + if let Some(registry) = owner { + crate::lifecycle_trace::reply( + registry.owner().raw(), + "Discharge", + slot as u64, + kind, + ); + } + crate::lifecycle_trace::downstream_closed(slot as u64, kind); + } + } + + /// A deferred token's deadline was lowered (`slot`'s lock held): + /// lifecycle traces (the model's `Shorten`). + #[inline(always)] + #[cfg_attr(not(feature = "lifecycle-trace"), allow(unused_variables))] + pub(crate) fn shortened(owner: &SlotOwner, slot: usize) { + #[cfg(feature = "lifecycle-trace")] + if let Some(registry) = owner { + crate::lifecycle_trace::reply(registry.owner().raw(), "Shorten", slot as u64, ""); + } + } + + /// A blocking receiver registered its unparker and is about to park: + /// the failpoint site `reply.recv_parked`, a handshake for tests. + #[inline(always)] + pub(crate) fn parked() { + fail_point!("reply.recv_parked"); + } + + /// Run `f` with a waker that unparks the current thread (cached per + /// thread: no allocation, and no reference count change when it is + /// already the slot's registered waker). + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + struct Unpark(std::thread::Thread); + impl std::task::Wake for Unpark { + fn wake(self: Arc) { + self.0.unpark(); + } + fn wake_by_ref(self: &Arc) { + self.0.unpark(); + } + } + thread_local! { + static WAKER: std::task::Waker = + std::task::Waker::from(Arc::new(Unpark(std::thread::current()))); + } + WAKER.with(|waker| f(waker)) + } + + /// Park the current thread for at most `ticks` clock ticks + /// (nanoseconds), or until unparked when `None`. + pub(crate) fn park(ticks: Option) { + match ticks { + None => std::thread::park(), + Some(ticks) => std::thread::park_timeout(Duration::from_nanos(ticks)), + } + } + use std::time::Duration; +} + +/// Clock ticks now, on the `spawned_rt` clock (virtual under simulation). +pub(crate) fn now_ticks() -> u64 { + clock::now() +} + +/// The clock deadlines are fixed against: one seam, so the source (today +/// the precise `spawned_rt` clock) can change without touching the reply +/// protocol. Ticks are nanoseconds on the `spawned_rt` epoch; `NEVER` is +/// reserved. +pub(crate) mod clock { + use super::NEVER; + use std::time::Duration; + + /// Ticks now. + #[inline] + pub(crate) fn now() -> u64 { + spawned_rt::time::now_ticks() + } + + /// The deadline `timeout` from now, in ticks (`NEVER` if it overflows), + /// computed directly: no `Instant` arithmetic or conversion. + #[inline] + pub(crate) fn after(timeout: Duration) -> u64 { + u64::try_from(timeout.as_nanos()) + .ok() + .and_then(|nanos| now().checked_add(nanos)) + .filter(|ticks| *ticks < spawned_rt::time::MAX_TICKS) + .unwrap_or(NEVER) + } +} + +// --------------------------------------------------------------------------- +// Deadline +// --------------------------------------------------------------------------- + +/// A request's deadline, fixed when it is sent. +/// +/// `gen_server:call/3` takes a timeout or `infinity`; a request takes an +/// absolute instant or [`Deadline::Never`]. `request` uses +/// [`DEFAULT_REQUEST_TIMEOUT`](crate::response::DEFAULT_REQUEST_TIMEOUT) +/// after the send; `send_request` takes the deadline. Every call that takes +/// a deadline takes `impl Into`: a `Duration` (from now), an +/// `Instant` or a `Deadline`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Deadline { + /// Expire at this instant (on the `spawned_rt::time` clock). + At(Instant), + /// Never expire; the caller waits until the reply or the owner's death. + Never, +} + +impl Deadline { + /// `timeout` from now. + pub fn after(timeout: Duration) -> Self { + match Instant::now().checked_add(timeout) { + Some(at) => Self::At(at), + None => Self::Never, + } + } + + pub(crate) fn ticks(self) -> u64 { + match self { + Self::At(at) => spawned_rt::time::to_ticks(at), + Self::Never => NEVER, + } + } + + fn of_ticks(ticks: u64) -> Self { + if ticks == NEVER { + Self::Never + } else { + Self::At(spawned_rt::time::from_ticks(ticks)) + } + } +} + +/// A timeout from now: [`Deadline::after`]. Every call that takes +/// `impl Into` takes a `Duration`, an `Instant` or a `Deadline`. +impl From for Deadline { + fn from(timeout: Duration) -> Self { + Self::after(timeout) + } +} + +/// An absolute instant on the `spawned_rt::time` clock. +impl From for Deadline { + fn from(at: Instant) -> Self { + Self::At(at) + } +} + +fn record(_delivery: Delivery) { + #[cfg(feature = "metrics")] + crate::instrumentation::reply(_delivery == Delivery::Delivered); +} + +/// The caller's error for a terminal slot. `Stopped` carries the owner's +/// final reason, which its registry stores at the death point before any of +/// its slots becomes `STOPPED`. +/// +/// A slot without an owner (a test double's `reply_pair`) has no actor whose +/// reason it could carry: it reports `NoProc`, as OTP's call to a process +/// that does not exist. `Abandoned` and an already taken `Replied` cannot +/// reach a caller (the receiver is gone, or consumed); they map the same way. +fn terminal_error(terminal: Terminal, slot: &Slot) -> ActorError { + match terminal { + Terminal::Dropped => ActorError::ReplyDropped, + Terminal::Overloaded => ActorError::Overloaded, + Terminal::TimedOut => ActorError::RequestTimeout, + Terminal::Stopped | Terminal::Abandoned | Terminal::Replied => ActorError::ActorStopped( + slot.stopped_reason() + .unwrap_or(crate::error::ExitReason::NoProc), + ), + } +} + +/// A new reply slot for a request with `deadline`, and the caller's receiver. +/// +/// `owner` is the receiving instance's registry. The slot takes no +/// reference to it (no shared reference count per request): the owner's +/// close records the final reason in the slot, and an undispatched +/// envelope is disposed of through the owner's mailbox +/// ([`SlotHandle::dispose`]). A `raw` request with a deadline (the public +/// `send_request`) records the owner's registry key in the slot's word, so +/// its receiver, awaited directly and still pending, puts the deadline in +/// that owner's heap without a runtime timer (a request answered before +/// that never touches the heap); the `request` paths arm their own timer +/// and pass `raw = false`. +pub(crate) fn reply_slot( + deadline: u64, + owner: Option<&Arc>, + raw: bool, +) -> (SlotHandle, ReplyReceiver) { + #[cfg(feature = "lifecycle-trace")] + let slot_owner = owner.cloned(); + #[cfg(not(feature = "lifecycle-trace"))] + let slot_owner = (); + let key = match owner { + Some(owner) if raw && deadline != NEVER => owner.key(), + _ => 0, + }; + let slot = Arc::new(Slot::with_owner_key(deadline, slot_owner, key)); + #[cfg(feature = "lifecycle-trace")] + if let Some(owner) = owner { + crate::lifecycle_trace::reply(owner.owner().raw(), "Send", Arc::as_ptr(&slot) as u64, ""); + crate::lifecycle_trace::downstream_sent(owner.owner().raw(), Arc::as_ptr(&slot) as u64); + } + (SlotHandle(Some(slot.clone())), ReplyReceiver::new(slot)) +} + +// --------------------------------------------------------------------------- +// Envelope side +// --------------------------------------------------------------------------- + +/// The envelope's hold on its request's slot. Dropped undispatched (mailbox +/// disposal at death, a rejected enqueue), its request is answered +/// `ActorStopped` with the owner's final reason at the death point. +pub(crate) struct SlotHandle(Option>>); + +impl SlotHandle { + /// No caller: the message was sent, not requested. + pub(crate) fn detached() -> Self { + Self(None) + } + /// Whether no caller waits (a send). + #[cfg_attr(not(feature = "lifecycle-trace"), allow(dead_code))] + pub(crate) fn is_detached(&self) -> bool { + self.0.is_none() + } + pub(crate) fn take(&mut self) -> Option>> { + self.0.take() + } + /// Undispatched at its owner's death point (mailbox disposal): answered + /// with the final reason, through the owner's registry. + pub(crate) fn dispose(&mut self, owner: Option<&Registry>) { + if let (Some(owner), Some(slot)) = (owner, self.0.take()) { + owner.dispose(slot); + } + } +} + +/// A turn's caller, lent by its envelope for the turn: retained (one +/// reference) only if the runtime destroys the turn unfinished, for the +/// death point to answer. +#[cfg_attr(not(feature = "async"), allow(dead_code))] +pub(crate) trait Caller: Sync { + fn retain(&self) -> Arc; +} + +impl Caller for Arc> { + fn retain(&self) -> Arc { + self.clone() + } +} + +/// Dropped undisposed (a refused admission, whose caller is answered +/// `NoProc` by the send itself): closed `STOPPED` now (trace builds, whose +/// slots know their owner, dispose of it there). +impl Drop for SlotHandle { + #[inline] + fn drop(&mut self) { + if let Some(slot) = self.0.take() { + #[cfg(feature = "lifecycle-trace")] + if let Some(owner) = slot.owner.clone() { + owner.dispose(slot); + return; + } + slot.close(); + } + } +} + +// --------------------------------------------------------------------------- +// PendingReply +// --------------------------------------------------------------------------- + +/// Why [`PendingReply::defer`] did not return a token. More reasons may be +/// added. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum DeferError { + /// The owner's `MAX_OUTSTANDING` credits are in use; the caller has + /// already been answered `Overloaded`. + #[error("deferred replies at the bound; the caller was answered Overloaded")] + Overloaded, + /// The request is no longer open: its deadline passed or its caller left. + #[error("the request is closed (expired or abandoned)")] + Closed, + /// The message was sent, not requested: there is no caller to answer. + #[error("the message was sent, not requested")] + NoCaller, +} + +/// The reply obligation of the request being handled, scoped to the turn. +/// +/// Answer it with [`reply`](Self::reply), hand it out of the turn with +/// [`defer`](Self::defer), or drop it: a request whose `PendingReply` is +/// dropped is answered `ReplyDropped` when the turn returns, or, when it +/// panics, `ActorStopped` with the final reason at the actor's death point. A message reached by `send` has a detached +/// `PendingReply`: `reply` drops the value and `defer` returns +/// [`DeferError::NoCaller`]. +pub struct PendingReply<'t, R: Send + 'static> { + /// Lent by the dispatching envelope for the turn (tasks mode), or owned. + slot: Option>>>, + registry: Option<&'t Arc>, + _turn: PhantomData<&'t mut ()>, +} + +impl std::fmt::Debug for PendingReply<'_, R> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingReply") + .field("detached", &self.slot.is_none()) + .finish_non_exhaustive() + } +} + +impl<'t, R: Send + 'static> PendingReply<'t, R> { + pub(crate) fn new(slot: Option>>, registry: &'t Arc) -> Self { + Self::with_slot(slot.map(Cow::Owned), registry) + } + + /// The obligation of a request whose slot the dispatching envelope + /// keeps for the turn: no reference is taken unless it is deferred or + /// dropped unanswered. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn lent(slot: Option<&'t Arc>>, registry: &'t Arc) -> Self { + Self::with_slot(slot.map(Cow::Borrowed), registry) + } + + fn with_slot(slot: Option>>>, registry: &'t Arc) -> Self { + #[cfg(feature = "lifecycle-trace")] + if let Some(slot) = &slot { + crate::lifecycle_trace::reply( + registry.owner().raw(), + "Dispatch", + Arc::as_ptr(&**slot) as u64, + "", + ); + } + Self { + slot, + registry: Some(registry), + _turn: PhantomData, + } + } + + /// A `PendingReply` for a sent message: there is no caller. For tests + /// calling a handler by hand, `spawned_test::detached_reply()`. + pub(crate) fn detached() -> Self { + Self { + slot: None, + registry: None, + _turn: PhantomData, + } + } + + /// Answer the caller now. A value that is not delivered is dropped. + #[inline] + pub fn reply(self, value: R) -> Delivery { + // Consumed here: the obligation's `Drop` (an unanswered reply) has + // nothing to do once the slot is taken, so it is skipped. + let mut this = std::mem::ManuallyDrop::new(self); + match this.slot.take() { + None => { + drop(value); + Delivery::CallerGone + } + Some(slot) => { + let (delivery, unsent) = slot.reply_in_turn(value); + drop(unsent); + record(delivery); + delivery + } + } + } + + /// Answer the caller `Err(Overloaded)` now. + pub fn overloaded(mut self) { + if let Some(slot) = self.slot.take() { + slot.terminate(Terminal::Overloaded); + } + } + + /// Take the obligation out of the turn as a storable [`ReplyTo`]. This + /// takes one of the owner's `MAX_OUTSTANDING` credits; at the bound the + /// caller is answered `Overloaded` and this returns + /// [`DeferError::Overloaded`]. + pub fn defer(mut self) -> Result, DeferError> { + let slot = self.slot.take().ok_or(DeferError::NoCaller)?.into_owned(); + let registry = self.registry.take().ok_or(DeferError::NoCaller)?; + match slot.defer(registry.pool(), &now_ticks) { + Ok(()) => { + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::downstream_deferred(Arc::as_ptr(&slot) as u64); + registry.register(&slot); + registry.track_deadline(&slot); + Ok(ReplyTo { + slot: Some(slot), + registry: registry.clone(), + stamp: crate::instrumentation::hooks::deferred(), + }) + } + Err(slot::Refused::Closed) => Err(DeferError::Closed), + Err(slot::Refused::Overloaded) => Err(DeferError::Overloaded), + } + } + + /// The request's deadline, fixed at send; [`Deadline::Never`] for an + /// unbounded request and for sends. Pass it on to forward the request + /// with the same deadline (`other.send_request(msg, reply.deadline())`). + pub fn deadline(&self) -> Deadline { + self.slot + .as_ref() + .map_or(Deadline::Never, |s| Deadline::of_ticks(s.deadline())) + } + + /// Whether the caller can no longer be answered (the deadline passed or + /// the caller left). Checks the deadline first. Always `true` for sends. + pub fn is_closed(&self) -> bool { + self.slot.as_ref().is_none_or(|s| s.is_closed(&now_ticks)) + } + + /// Answer `ActorStopped` (test doubles standing in for an absent actor: + /// without an owning actor the reason is `NoProc`). + pub(crate) fn discharge_stopped(mut self) { + if let Some(slot) = self.slot.take() { + slot.terminate(Terminal::Stopped); + } + } +} + +impl Drop for PendingReply<'_, R> { + #[inline] + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + if slot.is_terminal() { + return; + } + match self.registry { + Some(registry) => registry.dropped(slot.into_owned()), + None => { + slot.terminate(Terminal::Dropped); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// ReplyTo +// --------------------------------------------------------------------------- + +/// A deferred reply obligation: answer it from a later turn, a job, or +/// `stopped()`. Not `Clone`; `Send + 'static`. +/// +/// Dropping it answers the caller `ReplyDropped`; while the owner is exiting +/// it is held instead and answered at the death point. The owner's death +/// answers every open token `ActorStopped` with the final exit reason, +/// wherever it is. +#[must_use = "dropping a ReplyTo answers ReplyDropped (or ActorStopped at death while the owner exits)"] +pub struct ReplyTo { + slot: Option>>, + registry: Arc, + /// When it was deferred (sampled; zero-sized without `metrics`). + stamp: crate::instrumentation::hooks::DeferStamp, +} + +impl std::fmt::Debug for ReplyTo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyTo") + .field("owner", &self.registry.owner()) + .finish_non_exhaustive() + } +} + +impl ReplyTo { + /// Answer the caller. Returns how the reply was delivered. + pub fn reply(mut self, value: R) -> Delivery { + let slot = self + .slot + .take() + .expect("ReplyTo holds its slot until consumed"); + // Admitted by the owner's death-point fence: once `NoProc` is + // observable no reply succeeds (the owner's close answers the slot). + let (delivery, unsent) = slot.reply_admitted(value, || self.registry.reply_permit()); + drop(unsent); + record(delivery); + if delivery == Delivery::Delivered { + crate::instrumentation::hooks::deferred_replied(&self.stamp); + } + delivery + } + + /// Answer the caller `Err(Overloaded)`, e.g. on `InsertError::Full`. + /// Past the owner's death point the caller gets its final reason. + pub fn overloaded(mut self) { + if let Some(slot) = self.slot.take() { + // The permit covers the transition only, never its wakes. + slot.terminate_admitted(Terminal::Overloaded, || self.registry.reply_permit()); + } + } + + /// The request's deadline (as shortened so far). + pub fn deadline(&self) -> Deadline { + Deadline::of_ticks(self.slot().deadline()) + } + + /// Shorten the request's deadline to `deadline` (anything + /// `Into`: a `Duration` from now, an `Instant`, or a + /// [`Deadline`], e.g. another request's); a later deadline, or + /// [`Deadline::Never`], leaves it unchanged. The waiting caller is woken to re-arm its timer. + pub fn shorten_deadline(&self, deadline: impl Into) { + // Only a lowered deadline gets a heap entry, so repeated calls with + // the same or a later deadline store nothing. + if self.slot().shorten(deadline.into().ticks(), &now_ticks) { + self.registry.track_deadline(self.slot()); + } + } + + /// Whether the caller can no longer be answered. Checks the deadline + /// first. + pub fn is_closed(&self) -> bool { + self.slot().is_closed(&now_ticks) + } + + /// Wait until the caller can no longer be answered (expired, abandoned, + /// answered elsewhere or closed by the owner's death). + #[cfg(feature = "async")] + pub async fn closed(&self) { + let slot = self.slot(); + loop { + // One wait per deadline value: a shortening wakes the watchers, + // and the loop re-arms for the earlier deadline. + let deadline = slot.deadline(); + let mut woken = false; + let waiting = std::future::poll_fn(|cx| { + if woken && slot.deadline() != deadline { + return std::task::Poll::Ready(false); + } + woken = true; + slot.poll_closed(cx.waker(), &now_ticks).map(|()| true) + }); + let closed = if deadline == NEVER { + waiting.await + } else { + let wait = Duration::from_nanos(deadline.saturating_sub(now_ticks())); + match spawned_rt::tasks::timeout(wait, waiting).await { + Ok(closed) => closed, + Err(_) => { + slot.check_deadline(&now_ticks); + slot.is_terminal() + } + } + }; + if closed { + return; + } + } + } + + /// The actor instance that owns this obligation. + pub fn owner(&self) -> ActorId { + self.registry.owner() + } + + fn slot(&self) -> &Arc> { + self.slot + .as_ref() + .expect("ReplyTo holds its slot until consumed") + } + + pub(crate) fn slot_arc(&self) -> &Arc> { + self.slot() + } +} + +impl Drop for ReplyTo { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + self.registry.dropped(slot); + } + } +} + +// --------------------------------------------------------------------------- +// ReplyReceiver +// --------------------------------------------------------------------------- + +/// The caller's side of a request: `.await` it in `tasks` mode (it is +/// [`IntoFuture`](std::future::IntoFuture)), or call [`recv`](Self::recv) in +/// `threads` mode. Yields `Ok(reply)`, or `RequestTimeout` at the deadline +/// fixed at send, `ReplyDropped`, `Overloaded`, or `ActorStopped(reason)` +/// with the owner's final exit reason when it died first. +/// +/// One pointer wide: the deadline timer lives in the awaiting future +/// (`ReplyFuture`), not in the receiver, so receivers are cheap to store +/// and move. Dropping it abandons the request: a later reply is dropped and +/// any deferred credit returns at once. +pub struct ReplyReceiver { + slot: Option>>, +} + +impl Drop for ReplyReceiver { + #[inline] + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + slot.abandon(); + } + } +} + +impl std::fmt::Debug for ReplyReceiver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyReceiver") + .field("done", &self.slot.is_none()) + .finish_non_exhaustive() + } +} + +impl ReplyReceiver { + fn new(slot: Arc>) -> Self { + Self { slot: Some(slot) } + } + + /// Await with the caller's own runtime timer (the `request` and + /// `Response` paths, which follow the runtime's clock, paused test clocks + /// included). + #[cfg(feature = "async")] + pub(crate) fn into_timed_future(self) -> ReplyFuture { + ReplyFuture { + rx: self, + armed: NEVER, + first_ms: 0, + sleep: None, + timer: true, + tracked: false, + } + } + + /// [`into_timed_future`](Self::into_timed_future) for a request just + /// sent with `Deadline::after(timeout)`: its first timer sleeps + /// `timeout` (in whole milliseconds, rounded up: the runtime timer's + /// resolution) instead of reading the clock again. The request was sent + /// at most the send-to-poll gap ago, so the timer fires at or after the + /// deadline, never before. A deadline shortened meanwhile re-arms from + /// the clock. + #[cfg(feature = "async")] + pub(crate) fn into_timed_future_after(self, timeout: Duration) -> ReplyFuture { + let deadline = self.slot().deadline(); + // Whole milliseconds, rounded up, in 64-bit arithmetic (no 128-bit + // division); one that does not fit falls back to the clock. + let first_ms = timeout + .as_secs() + .checked_mul(1000) + .and_then(|ms| ms.checked_add(u64::from(timeout.subsec_nanos().div_ceil(1_000_000)))) + .and_then(|ms| u32::try_from(ms).ok()) + .unwrap_or(0); + ReplyFuture { + armed: deadline, + first_ms, + ..self.into_timed_future() + } + } + + fn slot(&self) -> &Arc> { + self.slot + .as_ref() + .expect("ReplyReceiver used after completion") + } + + /// The deadline fixed at send (as the owner shortened it so far). + pub fn deadline(&self) -> Deadline { + Deadline::of_ticks(self.slot().deadline()) + } + + /// Block the calling thread until the reply, the deadline or the owner's + /// death. Do not call it on an async executor thread. + pub fn recv(mut self) -> Result { + let slot = self + .slot + .take() + .expect("ReplyReceiver used after completion"); + slot.recv_blocking(&now_ticks) + .map_err(|terminal| terminal_error(terminal, &slot)) + } + + /// Non-blocking check: `None` while the request is still open. + pub fn try_recv(&mut self) -> Option> { + let slot = self.slot(); + let result = slot + .try_take(&now_ticks)? + .map_err(|terminal| terminal_error(terminal, slot)); + self.slot = None; + Some(result) + } +} + +#[cfg(feature = "async")] +impl std::future::IntoFuture for ReplyReceiver { + type Output = Result; + type IntoFuture = ReplyFuture; + fn into_future(self) -> ReplyFuture { + ReplyFuture { + rx: self, + armed: NEVER, + first_ms: 0, + sleep: None, + timer: false, + tracked: false, + } + } +} + +#[cfg(feature = "async")] +pin_project_lite::pin_project! { + /// Awaiting a [`ReplyReceiver`]: polls the slot and, while it is open, + /// sleeps until its deadline and then expires it. + pub struct ReplyFuture { + rx: ReplyReceiver, + armed: u64, + // A `request`'s timeout in milliseconds (0: none), with `armed` its + // send-time deadline: the first timer sleeps it (no clock read). + first_ms: u32, + // Arm a runtime timer (`request`/`Response`), rather than putting the + // deadline in the owner's heap (raw receivers). + timer: bool, + // The deadline is in the owner's heap. + tracked: bool, + #[pin] + sleep: Option, + } +} + +#[cfg(feature = "async")] +impl std::fmt::Debug for ReplyFuture { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReplyFuture").finish_non_exhaustive() + } +} + +#[cfg(feature = "async")] +impl std::future::Future for ReplyFuture { + type Output = Result; + + fn poll( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + use std::task::Poll; + let mut this = self.project(); + loop { + let slot = this.rx.slot(); + // Answered and no longer shared (the owner's turn already + // released its reference, the usual case once the caller gets + // to it): the value moves out without the slot lock. + if slot::state(slot.word()) == slot::REPLIED && Arc::strong_count(slot) == 1 { + let shared = this.rx.slot.take().expect("checked above"); + match Arc::try_unwrap(shared) { + Ok(owned) => { + return Poll::Ready( + owned + .into_reply() + .ok_or(ActorError::ActorStopped(crate::error::ExitReason::NoProc)), + ); + } + Err(shared) => this.rx.slot = Some(shared), + } + } + let slot = this.rx.slot(); + if let Poll::Ready(result) = slot.poll(cx.waker()) { + let result = result.map_err(|terminal| terminal_error(terminal, slot)); + this.rx.slot = None; + return Poll::Ready(result); + } + let deadline = slot.deadline(); + if deadline == NEVER { + return Poll::Pending; + } + // A raw request with a deadline puts it once in its owner's + // heap (found by the key in the word); the service expires the + // slot (waking this future) without a per-request runtime timer. + if !*this.timer { + let key = slot::owner_key(slot.word()); + if key != 0 { + if *this.tracked { + return Poll::Pending; + } + if let Some(owner) = Registry::by_key(key) { + owner.track_deadline(slot); + *this.tracked = true; + return Poll::Pending; + } + // The owner is gone: fall back to the runtime timer. + } + } + if *this.armed != deadline || this.sleep.is_none() { + let wait = match std::mem::take(this.first_ms) { + ms if ms != 0 && *this.armed == deadline => Duration::from_millis(ms.into()), + _ => Duration::from_nanos(deadline.saturating_sub(now_ticks())), + }; + this.sleep.set(Some(spawned_rt::tasks::sleep(wait))); + *this.armed = deadline; + } + let sleep = this.sleep.as_mut().as_pin_mut().expect("armed above"); + match sleep.poll(cx) { + Poll::Pending => return Poll::Pending, + // The timer was armed for the slot's deadline (which only + // decreases), so the deadline has passed: expire the slot. + // This follows the runtime's timer, so paused test clocks + // keep working. + Poll::Ready(()) => { + this.sleep.set(None); + this.rx.slot().terminate(Terminal::TimedOut); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Test helper +// --------------------------------------------------------------------------- + +/// A `PendingReply` not tied to an actor turn, and its receiver, for calling +/// a manual `Handler::handle` directly in tests (`spawned_test::reply_pair`). +/// Await the receiver only after driving the turns the reply depends on. +pub(crate) fn reply_pair( + deadline: Deadline, +) -> (PendingReply<'static, R>, ReplyReceiver) { + let (mut handle, receiver) = reply_slot(deadline.ticks(), None, false); + let reply = PendingReply::new(handle.take(), Registry::detached()); + (reply, receiver) +} diff --git a/concurrency/src/reply/pending.rs b/concurrency/src/reply/pending.rs new file mode 100644 index 000000000..1cb1c9c43 --- /dev/null +++ b/concurrency/src/reply/pending.rs @@ -0,0 +1,269 @@ +//! `PendingReplies`: a bounded table of deferred tokens by key +//! (docs/design/deferred-replies.md ยง7). + +use super::ReplyTo; +use std::hash::Hash; + +/// Why [`PendingReplies::insert`] handed the token back. The handler decides +/// the answer (typically `token.overloaded()` or a domain error) and still +/// owns the token. +/// +/// The enum is `#[non_exhaustive]`; match on [`kind`](Self::kind) (or take +/// both with [`into_parts`](Self::into_parts)) so the token is answered +/// whatever the reason, including one added later, instead of being +/// dropped in a wildcard arm (which answers the caller `ReplyDropped`): +/// +/// ```ignore +/// if let Err(refused) = table.insert(id, token) { +/// let (kind, token) = refused.into_parts(); +/// match kind { +/// InsertErrorKind::Duplicate => token.reply(Err(Error::DuplicateId)), +/// _ => token.overloaded(), // Full, and any later reason +/// } +/// } +/// ``` +#[derive(Debug)] +#[non_exhaustive] +pub enum InsertError { + /// A live token is already stored under this key; it is kept. + Duplicate(ReplyTo), + /// The table holds `cap` live tokens. + Full(ReplyTo), +} + +/// Why an insert was refused, without the token ([`InsertError::kind`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum InsertErrorKind { + /// A live token is already stored under this key. + Duplicate, + /// The table holds `cap` live tokens. + Full, +} + +impl InsertError { + /// Why the token was handed back. + pub fn kind(&self) -> InsertErrorKind { + match self { + Self::Duplicate(_) => InsertErrorKind::Duplicate, + Self::Full(_) => InsertErrorKind::Full, + } + } + /// The token handed back. + pub fn into_inner(self) -> ReplyTo { + match self { + Self::Duplicate(token) | Self::Full(token) => token, + } + } + /// The reason and the token together: every refusal can be answered. + pub fn into_parts(self) -> (InsertErrorKind, ReplyTo) { + (self.kind(), self.into_inner()) + } +} + +/// Deferred tokens by key, capped: retained entries never exceed `cap`, +/// however fast tokens expire. Terminal tokens (answered, expired, abandoned) +/// are purged on every insert and skipped by `take`. +/// +/// Experimental in 0.8: a utility shaped by one client (requests by id +/// answered when a peer responds); it may change before 1.0. +/// +/// Entries keep insertion order: [`drain`](Self::drain) yields them oldest +/// first, the same in every process, so answering a drained table is +/// reproducible under the simulator (no per-process hash seed decides it). +/// The table is a scan (insert already purges the whole table, and `cap` +/// bounds it): `take` and `insert` are linear in the held entries. +pub struct PendingReplies { + cap: usize, + tokens: Vec<(K, ReplyTo)>, +} + +impl std::fmt::Debug for PendingReplies { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PendingReplies") + .field("cap", &self.cap) + .field("retained", &self.tokens.len()) + .finish() + } +} + +impl PendingReplies { + pub fn with_capacity(cap: usize) -> Self { + Self { + cap, + tokens: Vec::new(), + } + } + + /// Purge terminal tokens, then insert. A duplicate key (with a live + /// token) or a full table hands the new token back. + pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError> { + self.purge(); + if self.tokens.iter().any(|(k, _)| *k == key) { + return Err(InsertError::Duplicate(token)); + } + if self.tokens.len() >= self.cap { + return Err(InsertError::Full(token)); + } + self.tokens.push((key, token)); + Ok(()) + } + + /// Remove and return the live token under `key`; a terminal one is + /// removed and skipped. + pub fn take(&mut self, key: &K) -> Option> { + let at = self.tokens.iter().position(|(k, _)| k == key)?; + let (_, token) = self.tokens.remove(at); + (!token.is_closed()).then_some(token) + } + + /// Remove every terminal token. + fn purge(&mut self) { + self.tokens.retain(|(_, token)| !token.is_closed()); + } + + /// Live (non-terminal) tokens held: a scan, so a token whose caller + /// just left no longer counts. + pub fn len(&self) -> usize { + self.tokens.iter().filter(|(_, t)| !t.is_closed()).count() + } + + /// Whether no live token is held. + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Take every live token, oldest first (for example to answer them in + /// `stopped()`). + pub fn drain(&mut self) -> impl Iterator)> + '_ { + self.tokens + .drain(..) + .filter(|(_, token)| !token.is_closed()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reply::{reply_pair, Deadline}; + use crate::ActorError; + + fn token() -> (ReplyTo, crate::reply::ReplyReceiver) { + let (reply, rx) = reply_pair::(Deadline::Never); + (reply.defer().unwrap(), rx) + } + + #[test] + fn bounded_with_duplicates_and_purge() { + crate::test_wait::bounded( + concat!(module_path!(), "::bounded_with_duplicates_and_purge"), + move || { + let mut table = PendingReplies::with_capacity(2); + let (a, ra) = token(); + let (b, rb) = token(); + let (c, rc) = token(); + let (dup, rdup) = token(); + table.insert(1, a).unwrap(); + table.insert(2, b).unwrap(); + let full = table.insert(3, c).unwrap_err(); + assert!(matches!(full, InsertError::Full(_))); + full.into_inner().overloaded(); + assert!(matches!(rc.recv(), Err(ActorError::Overloaded))); + let duplicate = table.insert(1, dup).unwrap_err(); + assert!(matches!(duplicate, InsertError::Duplicate(_))); + drop(duplicate); + assert!(matches!(rdup.recv(), Err(ActorError::ReplyDropped))); + // A caller that left frees its entry on the next insert. + drop(rb); + assert_eq!( + table.tokens.len(), + 2, + "terminal tokens stay until an insert" + ); + assert_eq!(table.len(), 1); + let (d, rd) = token(); + table.insert(4, d).unwrap(); + assert_eq!(table.tokens.len(), 2); + assert_eq!(table.len(), 2); + assert!(table.take(&2).is_none()); + table.take(&1).unwrap().reply(10); + table.take(&4).unwrap().reply(40); + assert_eq!(ra.recv().unwrap(), 10); + assert_eq!(rd.recv().unwrap(), 40); + }, + ); + } + /// #261: draining follows insertion order, so two tables built the + /// same way (in the old `HashMap`, two different `RandomState` seeds) + /// drain identically, oldest first, with taken and terminal entries + /// skipped. + #[test] + fn drain_follows_insertion_order() { + let keys: Vec = (0..64).map(|i| (i * 7_919) % 1_000).collect(); + let build = || { + let mut table = PendingReplies::with_capacity(keys.len()); + let mut receivers = Vec::new(); + for &k in &keys { + let (t, rx) = token(); + table.insert(k, t).unwrap(); + receivers.push(rx); + } + table.take(&keys[3]).unwrap().reply(0); + drop(receivers.remove(10)); // abandoned: terminal, skipped + (table, receivers) + }; + let (mut first, _a) = build(); + let (mut second, _b) = build(); + let once: Vec = first + .drain() + .map(|(k, t)| { + drop(t); + k + }) + .collect(); + let twice: Vec = second + .drain() + .map(|(k, t)| { + drop(t); + k + }) + .collect(); + let expected: Vec = keys + .iter() + .enumerate() + .filter(|(i, _)| *i != 3 && *i != 10) + .map(|(_, k)| *k) + .collect(); + assert_eq!(once, expected); + assert_eq!(twice, expected); + } + + /// Every refusal hands back its reason and its token together, so a + /// caller can answer the token for any reason, known or not. + #[test] + fn a_refusal_gives_its_kind_and_its_token() { + crate::test_wait::bounded( + concat!(module_path!(), "::a_refusal_gives_its_kind_and_its_token"), + move || { + let mut table = PendingReplies::with_capacity(1); + let (a, ra) = token(); + let (dup, rdup) = token(); + let (full, rfull) = token(); + table.insert(1, a).unwrap(); + let refused = table.insert(1, dup).unwrap_err(); + assert_eq!(refused.kind(), InsertErrorKind::Duplicate); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Duplicate); + token.overloaded(); + assert!(matches!(rdup.recv(), Err(ActorError::Overloaded))); + let refused = table.insert(2, full).unwrap_err(); + let (kind, token) = refused.into_parts(); + assert_eq!(kind, InsertErrorKind::Full); + token.reply(9); + assert_eq!(rfull.recv().unwrap(), 9); + table.take(&1).unwrap().reply(1); + assert_eq!(ra.recv().unwrap(), 1); + }, + ); + } +} diff --git a/concurrency/src/reply/registry.rs b/concurrency/src/reply/registry.rs new file mode 100644 index 000000000..a2d150c7b --- /dev/null +++ b/concurrency/src/reply/registry.rs @@ -0,0 +1,659 @@ +//! Per-instance reply registry (docs/design/deferred-replies.md ยง4, ยง5). +//! +//! The registry holds the owner's credit pool, a list of weak references to +//! its deferred tokens' slots, and a phase that classifies reply drops: +//! +//! | phase | a dropped reply obligation | +//! |---|---| +//! | `InTurn` | recorded on the turn; committed at turn end: `Dropped` if the handler returned, exit pending if it panicked | +//! | `Running` | `Dropped` at once | +//! | `Exiting` | exit pending | +//! | `Dead` | answered `Stopped` at once (the final reason is known) | +//! +//! **Exit pending** (docs/design/otp-exit-signals.md, "Required changes to +//! rank 11"): the slot stays `OPEN` and listed, and the death point answers +//! it with the owner's final reason, so callers, `Down` and `wait_exit` see +//! one reason. Only the caller's own deadline (or abandonment) completes it +//! earlier; nobody holds a token that could still reply. +//! +//! The final reason is stored once, at the death point ([`Registry::seal`]), +//! before any slot of this owner becomes `STOPPED`; the close copies it into +//! each slot it closes before the transition, and a caller that reads a +//! `STOPPED` slot reads it from there (a slot does not keep its owner alive). +//! +//! No rule uses `thread::panicking()`. The phase is one atomic so a turn costs +//! two uncontended atomic stores; the turn list is locked only when a drop +//! lands on it (`IN_TURN_PENDING`). + +use super::deadline::{self, DeadlineHeap, Expirable}; +use super::slot::{CreditPool, Slot, Terminal, Wake, NEVER}; +use crate::{ActorId, ExitReason}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, OnceLock, Weak}; + +/// Default `Actor::MAX_OUTSTANDING`: deferred credits per actor instance. +pub(crate) const DEFAULT_MAX_OUTSTANDING: usize = 1024; + +use super::turn::{Outcome, TurnPhase, DEAD, EXITING}; + +/// A slot seen without its reply type. +pub(crate) trait Closeable: Send + Sync { + /// Close without waking; the returned work runs with `run_wake`. + fn close_collect(&self) -> Option; + /// Record the owner's final reason before closing. + fn set_stopped_reason(&self, reason: &ExitReason); + fn run_wake(&self, wake: Wake); + fn notify_parked(&self, wake: &mut Wake); + fn terminate(&self, to: Terminal) -> bool; + fn is_terminal(&self) -> bool; +} + +/// A closed slot and its deferred wake-up. +/// A slot met by the death point's discharge, retained (whatever its state: +/// an unread reply's value is destroyed only after every caller is answered) +/// with the wake-up its close collected, if it closed it. +type PendingWake = (Arc, Option); + +impl Expirable for Slot { + fn pending_deadline(&self) -> Option { + let deadline = self.deadline(); + (!self.is_terminal() && deadline != NEVER).then_some(deadline) + } + fn expire(&self, now: u64) { + self.check_deadline(&|| now); + } +} + +impl Closeable for Slot { + fn close_collect(&self) -> Option { + Slot::close_collect(self) + } + fn set_stopped_reason(&self, reason: &ExitReason) { + Slot::set_stopped_reason(self, reason.clone()) + } + fn run_wake(&self, wake: Wake) { + Slot::run_wake(self, wake) + } + fn notify_parked(&self, wake: &mut Wake) { + Slot::notify_parked(self, wake) + } + fn terminate(&self, to: Terminal) -> bool { + Slot::terminate(self, to) + } + fn is_terminal(&self) -> bool { + Slot::is_terminal(self) + } +} + +#[derive(Default)] +struct Lists { + tokens: Vec>, + prune_at: usize, + deadlines: DeadlineHeap, +} + +pub(crate) struct Registry { + owner: ActorId, + /// Service key, unique per registry. + #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] + key: u64, + pool: Arc, + /// Turn phase and in-turn drops (its phase sits on its own cache line). + turn: TurnPhase>, + lists: Mutex, + /// When exit began: the cleanup deadline is armed from here. + exit_began: Mutex>, + /// The owner's final exit reason, set once at its death point. Set means + /// the mailbox refuses admission (`NoProc`). + final_reason: OnceLock, + /// Reply admission for deferred tokens answered off the owner's turn: + /// closed at the death point, before the final reason is visible, so no + /// reply succeeds once `NoProc` is observable. + replies: crate::mailbox::SendGate, + /// Exit-signal admission for this owner's contexts (`ctx.exit`): closed + /// at the death point, before the death notices are taken, so a signal + /// sent before death is delivered before them. + signals: crate::mailbox::SendGate, + /// While the death point discharges (from its close until + /// [`end_discharge`](Self::end_discharge)): the wake-ups of the slots it + /// closed, run only once every obligation is answered, so no waker runs + /// between two closes. + discharge: Mutex>>, + #[cfg(all(spawned_sim, feature = "async"))] + sim: deadline::SimService, +} + +fn lock(lists: &Mutex) -> std::sync::MutexGuard<'_, Lists> { + lists.lock().unwrap_or_else(|p| p.into_inner()) +} + +impl Registry { + pub(crate) fn new(owner: ActorId, max_outstanding: usize) -> Arc { + let max = max_outstanding.min(super::slot::MAX_JOBS as usize); + static NEXT_KEY: AtomicU64 = AtomicU64::new(1); + let registry = Arc::new(Self { + owner, + key: NEXT_KEY.fetch_add(1, Ordering::Relaxed), + pool: Arc::new(CreditPool::new(max)), + turn: TurnPhase::default(), + lists: Mutex::new(Lists { + prune_at: 16, + ..Lists::default() + }), + exit_began: Mutex::new(None), + final_reason: OnceLock::new(), + replies: crate::mailbox::SendGate::default(), + signals: crate::mailbox::SendGate::default(), + discharge: Mutex::new(None), + #[cfg(all(spawned_sim, feature = "async"))] + sim: deadline::SimService::default(), + }); + keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .insert(registry.key, Arc::downgrade(®istry)); + registry + } + + /// The live registry with this `key` (a raw request's slot records it). + #[cfg_attr(not(feature = "async"), allow(dead_code))] // awaited receivers only + pub(crate) fn by_key(key: u64) -> Option> { + let weak = keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .get(&key) + .cloned()?; + weak.upgrade() + } + + /// The registry of replies not tied to an actor (`reply_pair`): always + /// between turns, with the largest bound. + pub(crate) fn detached() -> &'static Arc { + static DETACHED: OnceLock> = OnceLock::new(); + DETACHED.get_or_init(|| Self::new(ActorId::next(), usize::MAX)) + } + + pub(crate) fn pool(&self) -> &Arc { + &self.pool + } + + pub(crate) fn owner(&self) -> ActorId { + self.owner + } + + /// Credits in use: live deferred tokens and running linked jobs. + pub(crate) fn outstanding(&self) -> usize { + self.pool.used() + } + + /// List a newly deferred token. Called by `defer()` after its CAS, in the + /// owner's turn, so it never races the close. + pub(crate) fn register(&self, slot: &Arc>) { + let weak: Weak = Arc::downgrade(slot) as Weak; + let mut lists = lock(&self.lists); + if lists.tokens.len() < lists.prune_at { + lists.tokens.push(weak); + return; + } + // Prune outside the lock: a temporary upgrade can become a slot's + // last strong reference, and dropping it drops an unread reply (user + // code, which may itself drop a token of this owner). + let listed = std::mem::take(&mut lists.tokens); + drop(lists); + let mut keep = Vec::with_capacity(listed.len()); + let mut upgraded = Vec::new(); + for token in listed { + if let Some(slot) = token.upgrade() { + if !slot.is_terminal() { + keep.push(token); + } + upgraded.push(slot); + } + } + let mut lists = lock(&self.lists); + keep.append(&mut lists.tokens); + lists.prune_at = (keep.len() * 2).max(16); + lists.tokens = keep; + lists.tokens.push(weak); + drop(lists); + drop(upgraded); + } + + #[cfg_attr(all(spawned_sim, feature = "async"), allow(dead_code))] + pub(crate) fn key(&self) -> u64 { + self.key + } + + #[cfg(all(spawned_sim, feature = "async"))] + pub(crate) fn sim_service(&self) -> &deadline::SimService { + &self.sim + } + + pub(crate) fn is_dead(&self) -> bool { + self.turn.load() == DEAD + } + + /// Put a deferred token's deadline in the owner's heap, so it expires + /// (and returns its credit) even when no caller polls. + pub(crate) fn track_deadline(self: &Arc, slot: &Arc>) { + let at = slot.deadline(); + if at == NEVER || slot.is_terminal() { + return; + } + let weak: Weak = Arc::downgrade(slot) as Weak; + self.track(at, weak); + } + + /// Add a heap entry; reschedule the service on a new earliest deadline. + pub(crate) fn track(self: &Arc, at: u64, item: Weak) { + let (earliest, upgraded) = { + let mut lists = lock(&self.lists); + if self.is_dead() { + return; + } + lists.deadlines.push(at, item) + }; + // Compaction's temporary upgrades are dropped outside the lock. + drop(upgraded); + if earliest { + deadline::schedule(self, at); + } + } + + /// Entries in the deadline heap, live and stale (bounded by compaction). + #[cfg(test)] + pub(crate) fn heap_entries(&self) -> usize { + lock(&self.lists).deadlines.len() + } + + /// The service visits: expire every due entry outside the lock, and + /// return the next earliest deadline. + /// + /// Entries run framework code that can reach user code (a waker, a + /// payload destructor the entry did not route to its failure sink, the + /// last reference to a job's `done`). Each runs contained, so one + /// failing entry neither skips the others nor ends the shared service. + pub(crate) fn service_deadlines(&self, now: u64) -> Option { + let due = lock(&self.lists).deadlines.due(now); + for item in due { + contain_entry(|| { + if item.pending_deadline().is_some_and(|d| d <= now) { + item.expire(now); + } + // Possibly the last reference: dropped inside the guard. + drop(item); + }); + } + lock(&self.lists).deadlines.earliest() + } + + /// A reply obligation (`PendingReply` or `ReplyTo`) was dropped unanswered. + #[inline] + pub(crate) fn dropped(&self, slot: Arc>) { + if slot.is_terminal() { + return; + } + let slot: Arc = slot; + if let Some((slot, outcome)) = self.turn.dropped(slot) { + self.commit(vec![slot], outcome); + } + } + + /// Answer `drops`: `Dropped` at once, or (`Stopped`) exit pending until + /// the death point. + fn commit(&self, drops: Vec>, outcome: Outcome) { + for slot in drops { + match outcome { + Outcome::Dropped => { + slot.terminate(Terminal::Dropped); + } + Outcome::Stopped => self.hold(slot), + } + } + } + + /// Exit pending: keep `slot` open and listed so the death point answers + /// it with the final reason. Past the death point, answer it now. + fn hold(&self, slot: Arc) { + let mut lists = lock(&self.lists); + // `close` sets `DEAD` before it takes the list under this lock, so a + // slot listed here is either taken by that close or closed below. + if self.is_dead() { + drop(lists); + self.close_slot(slot.clone()); + } else { + lists.tokens.push(Arc::downgrade(&slot)); + drop(lists); + } + // The caller's receiver may have been the other strong reference: + // dropping the last one runs no user code (the slot holds no value). + drop(slot); + } + + /// An undispatched envelope dropped its request (mailbox disposal, a + /// rejected enqueue): answered at the death point with the final reason. + pub(crate) fn dispose(&self, slot: Arc>) { + if !slot.is_terminal() { + self.hold(slot); + } + } + + /// Admit a deferred reply's linearization (its `OPEN -> REPLIED` CAS), + /// or `None` once the owner reached its death point: the slot then stays + /// listed and the registry's close answers it with the final reason. + pub(crate) fn reply_permit(&self) -> Option> { + self.replies.enter() + } + + /// Admit an exit signal sent by this owner, held until it is delivered; + /// `None` once the owner reached its death point (the signal is inert). + pub(crate) fn signal_permit(&self) -> Option> { + self.signals.enter() + } + + /// The death point's fences, before the final reason is visible: refuse + /// later replies and exit signals, and wait for those in flight (a CAS, + /// or one signal delivery: no user code). Never call it inside a + /// lifecycle-trace section: an in-flight delivery may need one. + pub(crate) fn fence(&self) { + self.replies.close(); + self.signals.close(); + } + + /// The death point D (called inside the owner's exit-table critical + /// section): fix the final reason callers will see and refuse admission. + /// Idempotent: the first reason stays. + pub(crate) fn seal(&self, reason: &ExitReason) { + let _ = self.final_reason.set(reason.clone()); + } + + /// The owner's final reason, once it reached its death point. + pub(crate) fn final_reason(&self) -> Option<&ExitReason> { + self.final_reason.get() + } + + /// Whether the owner is past its death point: sends and requests report + /// `NoProc`. + pub(crate) fn refuses_admission(&self) -> bool { + self.final_reason.get().is_some() + } + + /// The run loop is about to dispatch a handler. + #[inline] + pub(crate) fn begin_turn(&self) { + self.turn.begin_turn(); + } + + /// The handler returned (`panicked == false`) or unwound. + #[inline] + pub(crate) fn end_turn(&self, panicked: bool) { + let (drops, outcome) = self.turn.end_turn(panicked); + // Most turns drop no obligation: skip the commit entirely. + if !drops.is_empty() { + self.commit(drops, outcome); + } + } + + /// Exit begins: later drops, and the current turn's recorded drops, are + /// exit pending until the death point. + pub(crate) fn exiting(&self) { + if self.turn.load() == DEAD { + return; + } + let first = { + let mut began = self.exit_began.lock().unwrap_or_else(|p| p.into_inner()); + let first = began.is_none(); + began.get_or_insert_with(spawned_rt::time::Instant::now); + first + }; + self.commit(self.turn.exiting(), Outcome::Stopped); + if first { + crate::lifecycle_trace::reply(self.owner.raw(), "Exit", 0, ""); + } + } + + /// What remains of `timeout`, measured from the start of exit. + pub(crate) fn cleanup_remaining(&self, timeout: std::time::Duration) -> std::time::Duration { + let began = *self.exit_began.lock().unwrap_or_else(|p| p.into_inner()); + began.map_or(timeout, |began| timeout.saturating_sub(began.elapsed())) + } + + pub(crate) fn is_exiting(&self) -> bool { + self.turn.load() >= EXITING + } + + /// The death point starts discharging: closes from here until + /// [`end_discharge`](Self::end_discharge) defer their wake-ups. + pub(crate) fn begin_discharge(&self) { + let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); + batch.get_or_insert_with(Vec::new); + } + + /// Every obligation of the death point is answered (deferred slots and + /// the queued mailbox): run the collected wake-ups outside every lock, + /// parked blocking receivers first, so none waits behind an arbitrary + /// waker; then the wakers (executor or user code); then release the + /// retained slots (an unread reply's destructor is user code). Later + /// closes wake at once. Idempotent. + pub(crate) fn end_discharge(&self) { + let mut batch = self + .discharge + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + .unwrap_or_default(); + for (slot, wake) in &mut batch { + if let Some(wake) = wake { + slot.notify_parked(wake); + } + } + for (slot, wake) in &mut batch { + if let Some(wake) = wake.take() { + slot.run_wake(wake); + } + } + drop(batch); + } + + /// A turn destroyed unfinished (its runtime dropped the actor): its + /// caller is answered at the death point like every other one. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn answer_turn(&self, caller: Arc) { + self.close_slot(caller); + } + + /// Close one slot as `STOPPED`, deferring its wake-up while discharging. + /// While discharging, the slot itself is retained too, closed here or + /// already terminal: this may be the last strong reference to an unread + /// reply, whose destructor must not run before every caller is answered. + fn close_slot(&self, slot: Arc) -> bool { + // The final reason is sealed before any close: the caller reads it + // from the slot once it sees `STOPPED`. + if let Some(reason) = self.final_reason() { + slot.set_stopped_reason(reason); + } + let wake = slot.close_collect(); + let closed = wake.is_some(); + let mut batch = self.discharge.lock().unwrap_or_else(|p| p.into_inner()); + match batch.as_mut() { + Some(batch) => batch.push((slot, wake)), + None => { + drop(batch); + if let Some(wake) = wake { + slot.run_wake(wake); + } + drop(slot); + } + } + closed + } + + /// Death: every open token listed, and every exit-pending slot, becomes + /// `STOPPED` with the final reason, wherever it is. Returns how many were + /// closed. + /// + /// The run loops [`seal`](Self::seal) the reason at the death point + /// first. A registry closed without it (its lifecycle was dropped by the + /// runtime before the death point, G1) answers `Killed`. + pub(crate) fn close(&self) -> usize { + self.seal(&ExitReason::Killed); + crate::lifecycle_trace::reply(self.owner.raw(), "Death", 0, ""); + for slot in self.turn.close() { + self.close_slot(slot); + } + let tokens = std::mem::take(&mut lock(&self.lists).tokens); + let closed = tokens + .into_iter() + .filter_map(|t| t.upgrade()) + .filter(|s| { + fail_point!("reply.close_upgraded"); + self.close_slot(s.clone()) + }) + .count(); + // The heap is dropped after the close (ยง5 step 3.4), and the service + // forgets this owner, so dead actors leave nothing behind. + lock(&self.lists).deadlines.clear(); + deadline::forget(self); + #[cfg(all(spawned_sim, feature = "async"))] + self.sim.wake(); + closed + } +} + +/// Run one deadline-service step; a panic is logged and disposed of. +pub(crate) fn contain_entry(step: impl FnOnce()) { + if let Err(payload) = std::panic::catch_unwind(std::panic::AssertUnwindSafe(step)) { + let message = crate::error::panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + crate::actor_cleanup::log_safely(|| { + tracing::error!(event = "deadline_entry_failed", %message, + "a deadline entry panicked on the deadline service; servicing continues"); + }); + } +} + +/// Registries by key, for a raw receiver that finds its owner by the key in +/// its slot's word (one insert per actor instance, one removal at its drop). +fn keys() -> &'static Mutex>> { + static KEYS: OnceLock>>> = OnceLock::new(); + KEYS.get_or_init(Default::default) +} + +impl Drop for Registry { + fn drop(&mut self) { + let forgotten = keys() + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&self.key); + drop(forgotten); + // Every handle to the instance is gone; nothing can still answer. + if self.turn.load() != DEAD { + self.close(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::reply::slot::Slot; + + fn deferred(registry: &Arc, deadline: u64) -> Arc> { + let slot = Arc::new(Slot::new(deadline)); + slot.defer(registry.pool(), &|| 0).unwrap(); + registry.register(&slot); + registry.track_deadline(&slot); + slot + } + + fn terminal_of(slot: &Slot) -> Option { + Terminal::of(crate::reply::slot::state(slot.word())) + } + + #[test] + fn turn_drops_commit_dropped_or_wait_for_death() { + let registry = Registry::new(ActorId::next(), 8); + // A turn that returns answers its drops `Dropped`. + let slot = deferred(®istry, NEVER); + registry.begin_turn(); + registry.dropped(slot.clone()); + assert!( + !slot.is_terminal(), + "an in-turn drop waits for the turn end" + ); + registry.end_turn(false); + assert_eq!(terminal_of(&slot), Some(Terminal::Dropped)); + // Between turns a drop answers at once. + let slot = deferred(®istry, NEVER); + registry.dropped(slot.clone()); + assert_eq!(terminal_of(&slot), Some(Terminal::Dropped)); + // A panicking turn's drops, and drops while exiting, are exit + // pending: open until the death point answers them with the final + // reason. + let unwound = deferred(®istry, NEVER); + registry.begin_turn(); + registry.dropped(unwound.clone()); + registry.end_turn(true); + registry.exiting(); + let exiting = deferred(®istry, NEVER); + registry.dropped(exiting.clone()); + assert!(!unwound.is_terminal() && !exiting.is_terminal()); + registry.seal(&ExitReason::Shutdown); + assert_eq!(registry.close(), 2); + assert_eq!(terminal_of(&unwound), Some(Terminal::Stopped)); + assert_eq!(terminal_of(&exiting), Some(Terminal::Stopped)); + assert_eq!(registry.final_reason(), Some(&ExitReason::Shutdown)); + // Past the death point a drop answers at once. + let late = Arc::new(Slot::::new(NEVER)); + registry.dropped(late.clone()); + assert_eq!(terminal_of(&late), Some(Terminal::Stopped)); + assert_eq!(registry.outstanding(), 0); + } + + #[test] + fn unsealed_close_answers_killed() { + let registry = Registry::new(ActorId::next(), 8); + let open = deferred(®istry, NEVER); + assert_eq!(registry.close(), 1); + assert_eq!(terminal_of(&open), Some(Terminal::Stopped)); + assert_eq!(registry.final_reason(), Some(&ExitReason::Killed)); + } + + #[test] + fn close_stops_every_listed_open_token_once() { + let registry = Registry::new(ActorId::next(), 8); + let open = deferred(®istry, NEVER); + let answered = deferred(®istry, NEVER); + answered.reply(1); + assert_eq!(registry.close(), 1); + assert_eq!( + Terminal::of(crate::reply::slot::state(open.word())), + Some(Terminal::Stopped) + ); + assert_eq!(registry.outstanding(), 0); + assert_eq!(registry.heap_entries(), 0, "the heap is dropped at death"); + } + + #[test] + fn heap_stays_bounded_under_shortening_churn() { + let registry = Registry::new(ActorId::next(), 64); + let far = u64::MAX / 2; + let slots: Vec<_> = (0..32).map(|i| deferred(®istry, far + i)).collect(); + for round in 0..200u64 { + for slot in &slots { + // Each shortening supersedes the previous entry (stale). + slot.shorten(far - round - 1, &|| 0); + registry.track_deadline(slot); + } + assert!( + registry.heap_entries() <= 2 * slots.len() + 16 + slots.len(), + "heap grew to {} entries for {} live tokens", + registry.heap_entries(), + slots.len() + ); + } + // Expire them all: the service visit empties the heap of due entries. + assert_eq!(registry.service_deadlines(u64::MAX - 2), None); + assert_eq!(registry.outstanding(), 0); + } +} diff --git a/concurrency/src/reply/slot.rs b/concurrency/src/reply/slot.rs new file mode 100644 index 000000000..d51cc8f09 --- /dev/null +++ b/concurrency/src/reply/slot.rs @@ -0,0 +1,926 @@ +//! The reply slot: one packed atomic word per request +//! (docs/design/deferred-replies.md ยง1, "Slot synchronization protocol"). +//! +//! ```text +//! word: state:4 | TOKEN:1 | REG:1 | JOBS:16 | PARKED:1 | -:1 | OWNER:40 +//! credits(word) = max(JOBS, TOKEN and state is live ? 1 : 0) +//! ``` +//! +//! Every change of the state, `TOKEN`, `REG` or `JOBS` is one compare-and-swap +//! on the word. The owner's credit pool changes only by +//! `credits(new) - credits(old)` of a successful CAS: a positive delta is +//! reserved before the CAS and rolled back if the CAS fails; a negative delta +//! is returned after it, before the receiver is woken. +//! +//! A reply is one CAS `OPEN -> REPLIED` with the value written under the +//! slot lock; no reply reads the clock. The deadline is enforced only by an +//! expiry `OPEN -> TIMED_OUT` (the caller's timer, the owner's deadline +//! heap, or a `[deadline]` check), as OTP's caller-side `call` timeout: a +//! reply and an expiry race on the word and exactly one wins. +//! +//! This file is shared with the Loom model (`tests/reply_slot_loom.rs`), which +//! supplies the synchronization types through `super::sync`. Time is passed in +//! as a `now` closure returning clock ticks, so the model controls the clock. +//! No user code runs under a slot lock: values and wakers are moved out and +//! dropped or woken after it is released. +//! +//! The slot lock guards the value and the caller's waker (a thread blocked +//! in `recv_blocking` registers its unparker there: no condvar). Deferred +//! token state (the credit pool and the token's watchers) is created by +//! `defer()`, so a plain request never allocates it and its drop is cheap. +//! (An `AtomicWaker` beside a value mutex measured more instructions per +//! request than one lock for both: two more read-modify-writes.) + +use super::sync::{Arc, AtomicU64, AtomicUsize, Mutex, Ordering}; +use std::task::{Poll, Waker}; + +pub(crate) const OPEN: u64 = 0; +// State 1 was `WRITING` (a reply's deadline check between its CAS and its +// publication); replies no longer read the clock, so it no longer exists. +pub(crate) const REPLIED: u64 = 2; +pub(crate) const DROPPED: u64 = 3; +pub(crate) const OVERLOADED: u64 = 4; +pub(crate) const TIMED_OUT: u64 = 5; +pub(crate) const STOPPED: u64 = 6; +pub(crate) const ABANDONED: u64 = 7; + +const STATE_MASK: u64 = 0xF; +pub(crate) const TOKEN: u64 = 1 << 4; +pub(crate) const REG: u64 = 1 << 5; +const JOBS_SHIFT: u32 = 6; +const JOB_ONE: u64 = 1 << JOBS_SHIFT; +/// Largest running linked-job count per slot; `MAX_OUTSTANDING` is capped at +/// this value at configuration so the field cannot overflow. +pub(crate) const MAX_JOBS: u64 = 0xFFFF; +const JOBS_MASK: u64 = MAX_JOBS << JOBS_SHIFT; +/// The registered waker unparks a thread blocked in `recv_blocking` (no +/// user code): the death point's discharge wakes it before any other +/// waker. Set under the slot lock, before the waker is stored; no +/// transition reads or clears it otherwise (`with_state`, `JOBS` arithmetic +/// and `credits` leave it alone). A word bit, not a field: it keeps the +/// slot within the allocator's small size classes. +pub(crate) const PARKED: u64 = 1 << (JOBS_SHIFT + 16); +/// The owner's registry key (`Registry::key`, nonzero) of a raw request +/// with a deadline, fixed at construction in the word's top bits (no +/// transition touches them): an awaited receiver finds the owner by it and +/// puts the deadline in the owner's heap on its first pending poll, without +/// the slot holding a reference to the owner. +const KEY_SHIFT: u32 = JOBS_SHIFT + 18; +/// Largest owner key the word holds; a larger one is not recorded. +pub(crate) const MAX_KEY: u64 = u64::MAX >> KEY_SHIFT; +#[inline] +#[cfg_attr(not(feature = "async"), allow(dead_code))] // awaited receivers only +pub(crate) fn owner_key(word: u64) -> u64 { + word >> KEY_SHIFT +} + +/// Deadline value meaning "never" (`Deadline::Never`). +pub(crate) const NEVER: u64 = u64::MAX; + +#[inline] +pub(crate) fn state(word: u64) -> u64 { + word & STATE_MASK +} +#[inline] +pub(crate) fn jobs(word: u64) -> u64 { + (word & JOBS_MASK) >> JOBS_SHIFT +} +#[inline] +fn live(state: u64) -> bool { + state == OPEN +} +/// Credits a slot holds in its owner's pool. +#[inline] +pub(crate) fn credits(word: u64) -> u64 { + let token = (word & TOKEN != 0 && live(state(word))) as u64; + jobs(word).max(token) +} +#[inline] +fn with_state(word: u64, to: u64) -> u64 { + (word & !STATE_MASK) | to +} + +/// Terminal outcome of a request, as the caller sees it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Terminal { + Replied, + Dropped, + Overloaded, + TimedOut, + Stopped, + Abandoned, +} + +/// The model's name (`DeferredReplies.tla`) of the terminal state a slot +/// leaves `OPEN` for; `REPLIED` is the reply's linearization point. +fn trace_name(state: u64) -> &'static str { + match state { + REPLIED => "ok", + DROPPED => "dropped", + OVERLOADED => "overloaded", + TIMED_OUT => "timeout", + STOPPED => "stopped", + ABANDONED => "abandoned", + _ => "unknown", + } +} + +impl Terminal { + pub(crate) fn of(state: u64) -> Option { + Some(match state { + REPLIED => Self::Replied, + DROPPED => Self::Dropped, + OVERLOADED => Self::Overloaded, + TIMED_OUT => Self::TimedOut, + STOPPED => Self::Stopped, + ABANDONED => Self::Abandoned, + _ => return None, + }) + } + fn state(self) -> u64 { + match self { + Self::Replied => REPLIED, + Self::Dropped => DROPPED, + Self::Overloaded => OVERLOADED, + Self::TimedOut => TIMED_OUT, + Self::Stopped => STOPPED, + Self::Abandoned => ABANDONED, + } + } +} + +/// Outcome of a reply, as the replier sees it. More outcomes may be added. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum Delivery { + /// The value was published to the caller. + Delivered, + /// The caller left (abandoned the receiver) or the slot was already + /// answered (dropped, overloaded). + CallerGone, + /// The caller's deadline expired the request first (its timer, the + /// owner's deadline heap, or a deadline check); the value was dropped + /// and the caller sees `RequestTimeout`. A reply reads no clock (OTP's + /// `gen_server:call`): one that lands after the deadline but before + /// that expiry is `Delivered`. + Expired, + /// The owner died and its registry closed the slot (`ActorStopped`). + OwnerExited, +} + +/// Why `defer()` or a linked-job attach failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Refused { + /// The slot is no longer open (answered, expired, abandoned, closed). + Closed, + /// The owner's credit pool is exhausted. For `defer()` the caller has + /// already been answered `Overloaded`. + Overloaded, +} + +/// Owner-wide credit counter (`MAX_OUTSTANDING`). +pub(crate) struct CreditPool { + used: AtomicUsize, + max: usize, +} + +impl CreditPool { + pub(crate) fn new(max: usize) -> Self { + Self { + used: AtomicUsize::new(0), + max, + } + } + /// Reserve `n` credits, failing at the bound without waiting. + pub(crate) fn reserve(&self, n: u64) -> bool { + let n = n as usize; + self.used + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |used| { + used.checked_add(n).filter(|total| *total <= self.max) + }) + .is_ok() + } + pub(crate) fn release(&self, n: u64) { + if n != 0 { + let previous = self.used.fetch_sub(n as usize, Ordering::AcqRel); + debug_assert!(previous >= n as usize, "credit pool underflow"); + } + } + pub(crate) fn used(&self) -> usize { + self.used.load(Ordering::Acquire) + } + #[allow(dead_code)] // used by the Loom model + pub(crate) fn max(&self) -> usize { + self.max + } +} + +/// Rarely needed state, created on demand: by `defer()` (which installs the +/// pool before its CAS can take a credit) and by the owner's close at its +/// death point (the final reason). A plain request never allocates it. +#[derive(Default)] +struct Side { + /// The owner's pool, where this slot's credits return. + pool: Option>, + /// Token-side waiters: `ReplyTo::closed()` and caller-gone job + /// cancellation. Several may wait at once. + watchers: Vec, + /// The owner's final exit reason, stored by its close before the slot + /// becomes `STOPPED`, so the caller reads it from the slot. + reason: Option, +} + +/// What the slot lock guards. Its drop is two `None` checks and a null +/// check for a plain request. +struct Inner { + /// Written only by the reply that won the word, before the CAS that + /// publishes `REPLIED`; taken by the receiver after it reads `REPLIED`. + value: Option, + /// The caller's receiver: an async task's waker, or the unparker of a + /// thread blocked in `recv_blocking` (then the word has `PARKED`). + waker: Option, + side: Option>, +} + +pub(crate) struct Slot { + /// The receiving instance (production: its registry), whose deadline + /// heap expires an awaited raw request. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) owner: super::sync::SlotOwner, + word: AtomicU64, + deadline: AtomicU64, + /// The slot lock: the value and waker handoff, and the order of a + /// deadline shortening against a reply's deadline read. + inner: Mutex>, +} + +#[inline] +fn lock(mutex: &Mutex) -> super::sync::MutexGuard<'_, T> { + mutex.lock().unwrap_or_else(|p| p.into_inner()) +} + +/// What a winning transition must do after its CAS (the credit has already +/// returned): wake the receiver and the token's watchers. +pub(crate) struct Wake { + waker: Option, + /// `waker` unparks a thread blocked in `recv_blocking`. + parked: bool, + watchers: Vec, +} + +impl Wake { + const NONE: Self = Self { + waker: None, + parked: false, + watchers: Vec::new(), + }; + /// Wake a parked blocking receiver (no user code). Done first, so a + /// parked `recv()` never waits behind an arbitrary waker. + fn notify_parked(&mut self) { + if std::mem::take(&mut self.parked) { + if let Some(waker) = self.waker.take() { + waker.wake(); + } + } + } + #[inline] + fn run(self) { + if let Some(waker) = self.waker { + waker.wake(); + } + if !self.watchers.is_empty() { + for watcher in self.watchers { + watcher.wake(); + } + } + } +} + +impl Slot { + /// A fresh `OPEN` slot whose deadline is `deadline` ticks (`NEVER` for none). + #[allow(dead_code)] // the Loom model and unit tests build ownerless slots + #[allow(clippy::unit_arg)] // the owner is `()` outside trace builds + pub(crate) fn new(deadline: u64) -> Self { + Self::with_owner(deadline, Default::default()) + } + + pub(crate) fn with_owner(deadline: u64, owner: super::sync::SlotOwner) -> Self { + Self::with_owner_key(deadline, owner, 0) + } + + /// A slot whose word records the owner's registry `key` (0: none). + pub(crate) fn with_owner_key(deadline: u64, owner: super::sync::SlotOwner, key: u64) -> Self { + let key = if key <= MAX_KEY { key } else { 0 }; + Self { + owner, + word: AtomicU64::new(OPEN | (key << KEY_SHIFT)), + deadline: AtomicU64::new(deadline), + inner: Mutex::new(Inner { + value: None, + waker: None, + side: None, + }), + } + } + + #[inline] + pub(crate) fn word(&self) -> u64 { + self.word.load(Ordering::Acquire) + } + + #[inline] + pub(crate) fn deadline(&self) -> u64 { + self.deadline.load(Ordering::Acquire) + } + + #[inline] + pub(crate) fn is_terminal(&self) -> bool { + !live(state(self.word())) + } + + /// One CAS with credit accounting. `step` maps the current word to the new + /// one or aborts. A positive delta is reserved from `pool` first and rolled + /// back on CAS failure; `Err(None)` reports that the reservation failed. + /// Returns `(old, new)`; the caller returns any negative delta. + fn cas( + &self, + pool: Option<&CreditPool>, + mut step: impl FnMut(u64) -> Result, + ) -> Result<(u64, u64), Option> { + let mut current = self.word.load(Ordering::Acquire); + loop { + let next = step(current).map_err(Some)?; + let up = credits(next).saturating_sub(credits(current)); + if up > 0 { + match pool { + Some(pool) if pool.reserve(up) => {} + _ => { + // Report the bound only for the word it was computed + // from: a word that changed meanwhile (closed, + // abandoned) is re-evaluated, so a closed slot reports + // `Closed`, not `Overloaded`. + let actual = self.word.load(Ordering::Acquire); + if actual != current { + current = actual; + continue; + } + return Err(None); + } + } + } + let exchanged = super::sync::section(|| { + let exchanged = + self.word + .compare_exchange(current, next, Ordering::AcqRel, Ordering::Acquire); + if exchanged.is_ok() && state(current) == OPEN && state(next) != OPEN { + super::sync::left_open( + &self.owner, + self as *const Self as usize, + trace_name(state(next)), + current & TOKEN != 0, + ); + } + exchanged + }); + match exchanged { + Ok(_) => return Ok((current, next)), + Err(actual) => { + if let (Some(pool), true) = (pool, up > 0) { + pool.release(up); + } + current = actual; + } + } + } + } + + /// After a winning CAS `old -> new`: return the credit delta now and, + /// when the state became terminal, collect the wake-up, which the caller + /// runs after every lock is released. + #[inline] + fn after(&self, old: u64, new: u64) -> Wake { + let release = credits(old).saturating_sub(credits(new)); + let terminal = live(state(old)) && !live(state(new)); + if release == 0 && !terminal { + return Wake::NONE; + } + let mut inner = lock(&self.inner); + Self::finish(&mut inner, release, terminal, new) + } + + /// Under the slot lock: return `release` credits (an atomic, no user + /// code) and, for a terminal transition, take the wake-up. + #[inline] + fn finish(inner: &mut Inner, release: u64, terminal: bool, new: u64) -> Wake { + if release > 0 { + if let Some(pool) = inner.side.as_ref().and_then(|side| side.pool.as_ref()) { + pool.release(release); + } + } + if !terminal { + return Wake::NONE; + } + Wake { + waker: inner.waker.take(), + // `PARKED` is set under this lock before a thread stores its + // waker, so a CAS that raced it failed and retried with it. + parked: new & PARKED != 0, + watchers: match &mut inner.side { + Some(side) => std::mem::take(&mut side.watchers), + None => Vec::new(), + }, + } + } + + /// `OPEN -> to` (drop, overloaded, expiry, abandonment, registry close). + /// Returns whether this call won the transition. + pub(crate) fn terminate(&self, to: Terminal) -> bool { + self.terminate_admitted(to, || Some(())) + } + + /// [`terminate`](Self::terminate) admitted by the owner: `admit`'s + /// permit covers only the transition's CAS; the credit return and the + /// wakes (executor code) run after it is released. Refused (`None`), the + /// slot is left to the owner's close. + pub(crate) fn terminate_admitted

( + &self, + to: Terminal, + admit: impl FnOnce() -> Option

, + ) -> bool { + debug_assert!(to != Terminal::Replied); + let Some(permit) = admit() else { + return false; + }; + let target = to.state(); + let transition = self.cas::<()>(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, target)) + } else { + Err(()) + } + }); + drop(permit); + match transition { + Ok((old, new)) => { + self.after(old, new).run(); + true + } + Err(_) => false, + } + } + + /// Registry close at the owner's death: every slot still `OPEN` becomes + /// `STOPPED`; one already answered is left as it is. + pub(crate) fn close(&self) -> bool { + match self.close_collect() { + Some(wake) => { + wake.run(); + true + } + None => false, + } + } + + /// [`close`](Self::close) without the wake-up: the transition (and its + /// credit return) happens now and the caller runs the returned work with + /// [`run_wake`](Self::run_wake) once a whole batch is closed, so no + /// waker (executor or user code) runs between two closes. + pub(crate) fn close_collect(&self) -> Option { + let (old, new) = self + .cas::<()>(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, STOPPED)) + } else { + Err(()) + } + }) + .ok()?; + // The credit returns with the transition; only wakes wait. + Some(self.after(old, new)) + } + + /// The owner's close records its final reason before the `STOPPED` + /// transition can publish it (a caller reads it after seeing `STOPPED`). + pub(crate) fn set_stopped_reason(&self, reason: super::sync::Reason) { + if self.is_terminal() { + return; + } + lock(&self.inner) + .side + .get_or_insert_with(Default::default) + .reason = Some(reason); + } + + /// The final reason the owner's close recorded, if it closed this slot. + #[allow(dead_code)] // the Loom model has no reasons to read + pub(crate) fn stopped_reason(&self) -> Option { + lock(&self.inner) + .side + .as_ref() + .and_then(|side| side.reason.clone()) + } + + /// Run the wake-up work [`close_collect`](Self::close_collect) returned. + pub(crate) fn run_wake(&self, wake: Wake) { + wake.run(); + } + + /// The parked-receiver part of `wake`, run ahead of every waker of a + /// batch (see `Registry::end_discharge`). + pub(crate) fn notify_parked(&self, wake: &mut Wake) { + wake.notify_parked(); + } + + /// `[deadline]`: expire the slot if `now >= deadline`. Returns whether the + /// deadline has passed (whoever won the transition). + pub(crate) fn check_deadline(&self, now: &impl Fn() -> u64) -> bool { + let deadline = self.deadline(); + if deadline != NEVER && now() >= deadline { + self.terminate(Terminal::TimedOut); + true + } else { + false + } + } + + /// `is_closed()`: not `OPEN`, or past its deadline (then expired). + pub(crate) fn is_closed(&self, now: &impl Fn() -> u64) -> bool { + state(self.word()) != OPEN || self.check_deadline(now) + } + + /// `with_deadline(at)`: shorten only, then the `[deadline]` check. + /// Returns whether the deadline was lowered. + /// + /// A shortening changes the deadline only while the word is `OPEN` + /// (under the slot lock) and then runs the `[deadline]` check: an + /// earlier deadline already passed expires the slot at once, racing any + /// reply on the word (exactly one wins). Replies do not read it. + pub(crate) fn shorten(&self, at: u64, now: &impl Fn() -> u64) -> bool { + let (lowered, watchers, waker) = { + let mut inner = lock(&self.inner); + let lower = state(self.word()) == OPEN && at < self.deadline(); + if lower { + self.deadline.store(at, Ordering::Release); + super::sync::shortened(&self.owner, self as *const Self as usize); + // Moved out, not cloned: no executor code under the lock. A + // woken receiver re-registers when it polls again. + let watchers = match &mut inner.side { + Some(side) => std::mem::take(&mut side.watchers), + None => Vec::new(), + }; + (true, watchers, inner.waker.take()) + } else { + (false, Vec::new(), None) + } + }; + // Both sides re-arm their timers for the earlier deadline: the + // token's waiters and the caller's receiver (a parked `recv_blocking` + // thread recomputes its wait). + if let Some(waker) = waker { + waker.wake(); + } + for watcher in watchers { + watcher.wake(); + } + self.check_deadline(now); + lowered + } + + /// `defer()`: `OPEN, TOKEN=0 -> OPEN, TOKEN=1, REG=1`, reserving the + /// token's credit when no linked job holds one. At the bound the caller + /// is answered `Overloaded` in the same step. + pub(crate) fn defer( + &self, + pool: &Arc, + now: &impl Fn() -> u64, + ) -> Result<(), Refused> { + if self.check_deadline(now) { + return Err(Refused::Closed); + } + // The side state (the pool credits return to) exists before the CAS + // can take a credit. + lock(&self.inner) + .side + .get_or_insert_with(Default::default) + .pool + .get_or_insert_with(|| pool.clone()); + match self.cas(Some(pool), |w| { + if state(w) == OPEN && w & TOKEN == 0 { + Ok(w | TOKEN | REG) + } else { + Err(()) + } + }) { + Ok(_) => Ok(()), + Err(Some(())) => Err(Refused::Closed), + // At the bound the caller is answered `Overloaded` in the same + // step; if it closed first (abandoned, expired), this is `Closed`. + Err(None) if self.terminate(Terminal::Overloaded) => Err(Refused::Overloaded), + Err(None) => Err(Refused::Closed), + } + } + + /// `pipe_reply` attach: requires `OPEN` and `TOKEN`; `JOBS + 1`. The first + /// job shares the token's credit; a further one reserves a new credit. + pub(crate) fn attach(&self, now: &impl Fn() -> u64) -> Result<(), Refused> { + if self.check_deadline(now) { + return Err(Refused::Closed); + } + let pool = lock(&self.inner) + .side + .as_ref() + .and_then(|side| side.pool.clone()); + match self.cas(pool.as_deref(), |w| { + if state(w) != OPEN || w & TOKEN == 0 { + Err(Refused::Closed) + } else if jobs(w) >= MAX_JOBS { + Err(Refused::Overloaded) + } else { + Ok(w + JOB_ONE) + } + }) { + Ok(_) => Ok(()), + Err(Some(refused)) => Err(refused), + Err(None) => Err(Refused::Overloaded), + } + } + + /// A linked job ended (in any state): `JOBS - 1`, returning a credit if + /// this lowers `credits`. + pub(crate) fn job_end(&self) { + match self.cas::<()>(None, |w| { + debug_assert!(jobs(w) >= 1, "job_end without a running job"); + Ok(w - JOB_ONE) + }) { + Ok((old, new)) => self.after(old, new).run(), + Err(_) => unreachable!("job_end never aborts or reserves"), + } + } + + /// Reply: write the value under the slot lock and publish it with one + /// CAS `OPEN -> REPLIED`, the reply's linearization point (a receiver + /// reads the value under that lock). No clock is read: as in OTP's + /// `gen_server:call`, the caller's deadline is enforced only by the + /// caller's expiry (`OPEN -> TIMED_OUT`, its timer or the owner's deadline + /// heap); a reply that loses that race finds the slot `TIMED_OUT` and + /// gets its value back as `Expired`. A value that was not published is + /// returned so the caller drops it outside every lock. + #[allow(dead_code)] // unit tests and the Loom model + pub(crate) fn reply(&self, value: R) -> (Delivery, Option) { + self.reply_admitted(value, || Some(())) + } + + /// A reply from the owner's own turn (`PendingReply::reply`): the + /// owner's death point cannot run during its own turn, so no admission + /// is needed. Such a slot was never deferred (`defer()` consumes the + /// `PendingReply`), so it holds no credit, no job and no side state: + /// one CAS with the value written under the lock, then the receiver's + /// waker. Anything else (a lost race, an unexpected word) takes + /// [`publish`](Self::publish). + #[inline] + pub(crate) fn reply_in_turn(&self, value: R) -> (Delivery, Option) { + let mut inner = lock(&self.inner); + let current = self.word(); + if state(current) != OPEN || credits(current) != 0 || inner.side.is_some() { + drop(inner); + return self.publish(value, || Some(())); + } + inner.value = Some(value); + // `PARKED` changes only under this lock, and nothing else moves a + // word without credits but a terminal transition: a failed CAS + // means the slot closed first. + let published = super::sync::section(|| { + let exchanged = self.word.compare_exchange( + current, + with_state(current, REPLIED), + Ordering::AcqRel, + Ordering::Acquire, + ); + if exchanged.is_ok() { + super::sync::left_open( + &self.owner, + self as *const Self as usize, + trace_name(REPLIED), + false, + ); + } + exchanged + }); + if published.is_err() { + let value = inner.value.take().expect("written above"); + drop(inner); + return self.publish(value, || Some(())); + } + let waker = inner.waker.take(); + drop(inner); + if let Some(waker) = waker { + waker.wake(); + } + (Delivery::Delivered, None) + } + + /// [`reply`](Self::reply) admitted by the owner: `admit` returns a + /// permit held across the publishing CAS (no user code: a lock and a + /// value move), or `None` once the owner is past its death point. + /// Refused, the slot is left to the owner's close (`OwnerExited`). + pub(crate) fn reply_admitted

( + &self, + value: R, + admit: impl FnOnce() -> Option

, + ) -> (Delivery, Option) { + self.publish(value, admit) + } + + #[inline] + fn publish

(&self, value: R, admit: impl FnOnce() -> Option

) -> (Delivery, Option) { + let Some(permit) = admit() else { + return (Delivery::OwnerExited, Some(value)); + }; + let mut inner = lock(&self.inner); + inner.value = Some(value); + let published = self.cas(None, |w| { + if state(w) == OPEN { + Ok(with_state(w, REPLIED)) + } else { + Err(state(w)) + } + }); + drop(permit); + match published { + Ok((old, new)) => { + let wake = Self::finish( + &mut inner, + credits(old).saturating_sub(credits(new)), + true, + new, + ); + drop(inner); + wake.run(); + (Delivery::Delivered, None) + } + Err(lost) => { + let value = inner.value.take(); + drop(inner); + let delivery = match lost { + Some(TIMED_OUT) => Delivery::Expired, + Some(STOPPED) => Delivery::OwnerExited, + _ => Delivery::CallerGone, + }; + (delivery, value) + } + } + } + + /// Receiver side: take the outcome of a terminal word. `Replied` with no + /// value means it was already taken. + #[inline] + fn take(&self, state: u64) -> Option> { + let terminal = Terminal::of(state)?; + Some(match terminal { + Terminal::Replied => match lock(&self.inner).value.take() { + Some(value) => Ok(value), + None => Err(Terminal::Replied), + }, + other => Err(other), + }) + } + + /// Poll from the caller's receiver: registers `waker` and returns + /// `Pending` while the slot is open. The caller's timer expires the slot + /// at its deadline (`ReplyReceiver` sleeps until it), so the poll itself + /// reads no clock. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + #[inline] + pub(crate) fn poll(&self, waker: &Waker) -> Poll> { + if let Some(result) = self.take(state(self.word())) { + return Poll::Ready(result); + } + // `Waker::clone` and dropping a waker run executor code, so neither + // happens under the slot lock. The first registration clones before + // locking (one lock section instead of two); a later poll with the + // same waker finds it registered and drops the copy. + let mut fresh: Option = Some(waker.clone()); + loop { + if let Some(result) = self.take(state(self.word())) { + drop(fresh); + return Poll::Ready(result); + } + let mut inner = lock(&self.inner); + // Checked under the lock a terminal transition takes the waker + // under: a waker stored here is taken by it. + if !live(state(self.word())) { + continue; + } + if inner.waker.as_ref().is_some_and(|w| w.will_wake(waker)) { + drop(inner); + drop(fresh); + return Poll::Pending; + } + let Some(new) = fresh.take() else { + drop(inner); + fresh = Some(waker.clone()); + continue; + }; + let old = inner.waker.replace(new); + drop(inner); + drop(old); + return Poll::Pending; + } + } + + /// Token side: ready once the slot is terminal (or its deadline passed, + /// which expires it). Registers `waker` as a watcher otherwise. + #[cfg_attr(not(feature = "async"), allow(dead_code))] + pub(crate) fn poll_closed(&self, waker: &Waker, now: &impl Fn() -> u64) -> Poll<()> { + let mut fresh: Option = None; + loop { + if !live(state(self.word())) || self.check_deadline(now) { + return Poll::Ready(()); + } + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + continue; + } + let Some(side) = &mut inner.side else { + // Only a deferred token (or its linked job) watches its slot. + debug_assert!(false, "poll_closed on a slot that was never deferred"); + return Poll::Ready(()); + }; + if side.watchers.iter().any(|w| w.will_wake(waker)) { + drop(inner); + drop(fresh); + return Poll::Pending; + } + let Some(new) = fresh.take() else { + drop(inner); + fresh = Some(waker.clone()); + continue; + }; + side.watchers.push(new); + return Poll::Pending; + } + } + + /// Non-blocking check from the caller's receiver. + pub(crate) fn try_take(&self, now: &impl Fn() -> u64) -> Option> { + if let Some(result) = self.take(state(self.word())) { + return Some(result); + } + if self.check_deadline(now) { + return self.take(state(self.word())); + } + None + } + + /// Threads-mode receive: park until terminal, expiring the slot at its + /// deadline. + pub(crate) fn recv_blocking(&self, now: &impl Fn() -> u64) -> Result { + loop { + if let Some(result) = self.take(state(self.word())) { + return result; + } + if self.check_deadline(now) { + continue; + } + let deadline = self.deadline(); + let remaining = (deadline != NEVER).then(|| deadline.saturating_sub(now())); + if remaining == Some(0) { + continue; + } + { + let mut inner = lock(&self.inner); + if !live(state(self.word())) { + continue; + } + // Before the waker is stored (see `PARKED`). + self.word.fetch_or(PARKED, Ordering::AcqRel); + // The thread's own unparker (no user code to clone or drop). + super::sync::with_thread_waker(|waker| { + if !inner.waker.as_ref().is_some_and(|w| w.will_wake(waker)) { + inner.waker = Some(waker.clone()); + } + }); + } + super::sync::parked(); + // A stale unpark or a spurious return only loops. + super::sync::park(remaining); + } + } + + /// The reply of a `REPLIED` slot the caller now owns alone: moved out + /// without taking the lock (the publishing CAS, read by the caller, + /// ordered the value's write before it, and owning the slot excludes + /// every other access). `None` if it was already taken. + #[allow(dead_code)] // the Loom model takes through the lock + pub(crate) fn into_reply(self) -> Option { + self.inner + .into_inner() + .unwrap_or_else(|p| p.into_inner()) + .value + } + + /// The caller dropped its receiver: `OPEN -> ABANDONED`. + pub(crate) fn abandon(&self) -> bool { + self.terminate(Terminal::Abandoned) + } +} diff --git a/concurrency/src/reply/turn.rs b/concurrency/src/reply/turn.rs new file mode 100644 index 000000000..0c3068d3c --- /dev/null +++ b/concurrency/src/reply/turn.rs @@ -0,0 +1,148 @@ +//! The owner's turn phase and the commit of reply obligations dropped in a +//! turn (docs/design/deferred-replies.md ยง4, "Drop classification"). +//! +//! | phase | a dropped obligation | +//! |---|---| +//! | `InTurn` | recorded; committed at turn end: `Dropped` if the handler returned, `Stopped` if it panicked | +//! | `Running` | `Dropped` at once | +//! | `Exiting` / `Dead` | `Stopped` | +//! +//! `Stopped` means "answered with the owner's final reason": the registry +//! holds such a slot open (exit pending) until the death point, or closes it +//! at once when the owner is already dead. +//! +//! Shared with the Loom model (`tests/registry_turn_loom.rs`), which supplies +//! the synchronization types through `super::sync`. The protocol never runs +//! user code: it returns the items and their outcome, and the caller applies +//! them after every lock is released. + +use super::sync::{AtomicU8, Mutex, Ordering}; + +pub(crate) const RUNNING: u8 = 0; +pub(crate) const IN_TURN: u8 = 1; +pub(crate) const IN_TURN_PENDING: u8 = 2; +pub(crate) const EXITING: u8 = 3; +pub(crate) const DEAD: u8 = 4; + +/// How a dropped obligation is answered. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Outcome { + Dropped, + Stopped, +} + +/// On its own cache line: the run loop writes it every turn. +#[repr(align(64))] +struct Phase(AtomicU8); + +pub(crate) struct TurnPhase { + phase: Phase, + drops: Mutex>, +} + +impl Default for TurnPhase { + fn default() -> Self { + Self { + phase: Phase(AtomicU8::new(RUNNING)), + drops: Mutex::new(Vec::new()), + } + } +} + +fn take(drops: &Mutex>) -> Vec { + std::mem::take(&mut *drops.lock().unwrap_or_else(|p| p.into_inner())) +} + +impl TurnPhase { + #[inline] + pub(crate) fn load(&self) -> u8 { + self.phase.0.load(Ordering::Acquire) + } + + /// An obligation was dropped: `None` when it was recorded on the current + /// turn, otherwise the item and how to answer it now. + pub(crate) fn dropped(&self, item: T) -> Option<(T, Outcome)> { + loop { + match self.load() { + RUNNING => return Some((item, Outcome::Dropped)), + EXITING | DEAD => return Some((item, Outcome::Stopped)), + _ => { + let mut drops = self.drops.lock().unwrap_or_else(|p| p.into_inner()); + let mut current = self.load(); + let recorded = loop { + if current != IN_TURN && current != IN_TURN_PENDING { + break false; + } + match self.phase.0.compare_exchange( + current, + IN_TURN_PENDING, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(_) => break true, + Err(actual) => current = actual, + } + }; + if recorded { + drops.push(item); + return None; + } + // The turn ended first: classify by the new phase. + } + } + } + } + + /// The run loop is about to dispatch a handler. Only the run loop moves + /// the phase out of `RUNNING`, so a plain store suffices. + #[inline] + pub(crate) fn begin_turn(&self) { + self.phase.0.store(IN_TURN, Ordering::Release); + } + + /// The handler returned (`panicked == false`) or unwound: the recorded + /// drops and their outcome. + #[inline] + pub(crate) fn end_turn(&self, panicked: bool) -> (Vec, Outcome) { + let outcome = if panicked { + Outcome::Stopped + } else { + Outcome::Dropped + }; + if self.phase.0.swap(RUNNING, Ordering::AcqRel) == IN_TURN_PENDING { + (take(&self.drops), outcome) + } else { + (Vec::new(), outcome) + } + } + + /// Exit begins (idempotent; not after death): recorded drops commit as + /// `Stopped`. + pub(crate) fn exiting(&self) -> Vec { + let mut current = self.load(); + loop { + if current >= EXITING { + return Vec::new(); + } + match self.phase.0.compare_exchange( + current, + EXITING, + Ordering::AcqRel, + Ordering::Acquire, + ) { + Ok(previous) if previous == IN_TURN_PENDING => return take(&self.drops), + Ok(_) => return Vec::new(), + Err(actual) => current = actual, + } + } + } + + /// Death: recorded drops commit as `Stopped`. + pub(crate) fn close(&self) -> Vec { + if self.phase.0.swap(DEAD, Ordering::AcqRel) == IN_TURN_PENDING { + take(&self.drops) + } else { + Vec::new() + } + } +} diff --git a/concurrency/src/response.rs b/concurrency/src/response.rs index d235d0cc2..f3732bbed 100644 --- a/concurrency/src/response.rs +++ b/concurrency/src/response.rs @@ -1,6 +1,6 @@ use crate::error::ActorError; #[cfg(feature = "async")] -use spawned_rt::tasks::oneshot; +use crate::reply::{ReplyFuture, ReplyReceiver}; use std::future::Future; use std::pin::Pin; use std::time::Duration; @@ -38,10 +38,14 @@ impl RequestOptions { // --------------------------------------------------------------------------- enum ResponseState { + /// Sent, not yet polled: the deadline is already in the receiver's slot, + /// fixed at send. Awaiting arms the runtime timer; off a runtime, + /// `unwrap`, `expect`, `map` and `into_result` block on it instead. #[cfg(feature = "async")] - Receiver(oneshot::Receiver), + Sent(ReplyReceiver), + /// Being awaited. #[cfg(feature = "async")] - TimedReceiver(Pin> + Send>>), + Receiver(Pin>>), Ready(Result), Done, } @@ -49,8 +53,19 @@ enum ResponseState { /// Concrete wrapper for protocol request-response methods that works in both /// `tasks` (async) and `threads` (blocking) modes. /// -/// - **Tasks mode**: wraps a oneshot receiver; `.await` returns `Result` -/// - **Threads mode**: wraps a pre-computed result; use `.unwrap()` / `.expect()` directly +/// - **Tasks mode**: wraps the request's [receiver](crate::ReplyReceiver); +/// `.await` returns `Result`. The request's deadline was fixed +/// when it was sent. +/// - **Threads mode**: wraps a pre-computed result; use `.unwrap()`, +/// `.expect()` or `.into_result()` directly. +/// +/// A tasks-mode response that has not been awaited is *pending*. `unwrap`, +/// `expect`, `map` and `into_result` on a pending response block the calling +/// thread until the reply when called outside an async runtime (from plain +/// synchronous code), and panic inside one (blocking an executor thread can +/// deadlock the actor that must answer): there, `.await` it. `is_ok` and +/// `is_err` cannot wait (they borrow), so on a pending response they panic +/// rather than answer `false`. /// /// Protocol methods return `Response`: /// ```ignore @@ -67,11 +82,21 @@ pub struct Response( #[cfg(feature = "metrics")] Option, ); -// Safety: tokio::sync::oneshot::Receiver implements Unpin, so pinning -// Response is safe. This is required for the Future::poll implementation -// which uses self.get_mut(). +// The receiver is boxed and pinned, so `Response` is `Unpin` whatever `T` +// is; `Future::poll` uses `self.get_mut()`. impl Unpin for Response {} +/// Whether the calling thread runs an async runtime, where a pending +/// response must be awaited, not waited for. +#[cfg(feature = "async")] +fn on_async_runtime() -> bool { + #[cfg(not(spawned_sim))] + return spawned_rt::tasks::Handle::try_current().is_ok(); + // The simulator runs every task on one thread: blocking would deadlock. + #[cfg(spawned_sim)] + return true; +} + impl Response { /// Create a `Response` from a pre-computed result. /// @@ -85,115 +110,90 @@ impl Response { ) } - /// Extract the value, panicking on error. - /// - /// For ready (pre-computed) responses. Panics on pending responses โ€” - /// use `.await.unwrap()` in async contexts instead. - pub fn unwrap(self) -> T { - match self.0 { - ResponseState::Ready(result) => result.unwrap(), + /// The result: at once for a ready response; for a pending one, block + /// until the reply, the request's deadline or the actor's death (outside + /// an async runtime only; inside one this panics: `.await` it). + pub fn into_result(self) -> Result { + self.resolve("into_result") + } + + fn resolve(mut self, op: &str) -> Result { + match std::mem::replace(&mut self.0, ResponseState::Done) { + ResponseState::Ready(result) => result, #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { - panic!("called unwrap() on a pending Response; use .await in async contexts") + ResponseState::Sent(rx) => { + if on_async_runtime() { + panic!( + "called {op}() on a pending Response inside an async runtime; \ + .await it instead" + ); + } + let result = rx.recv(); + #[cfg(feature = "metrics")] + if let Some(observation) = self.1.take() { + observation.finish(&result); + } + result } - ResponseState::Done => panic!("Response already consumed"), + #[cfg(feature = "async")] + ResponseState::Receiver(_) => { + panic!("called {op}() on a Response that is being awaited; .await it") + } + ResponseState::Done => panic!("called {op}() on a Response already consumed"), } } - /// Extract the value, panicking with a custom message on error. + /// Extract the value, panicking on error. A pending response is waited + /// for (see [`Response`]). + pub fn unwrap(self) -> T { + self.resolve("unwrap").unwrap() + } + + /// Extract the value, panicking with a custom message on error. A + /// pending response is waited for (see [`Response`]). pub fn expect(self, msg: &str) -> T { - match self.0 { - ResponseState::Ready(result) => result.expect(msg), - #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { - panic!("{msg}: called expect() on a pending Response; use .await in async contexts") - } - ResponseState::Done => panic!("{msg}: Response already consumed"), - } + self.resolve("expect").expect(msg) } - /// Returns `true` if the response contains `Ok`. - /// Only meaningful for ready responses. - /// Returns `false` for pending (`Receiver`) or consumed (`Done`) states. + /// Returns `true` if the response is ready and `Ok`. Panics on a + /// pending response (it cannot wait through a borrow; use + /// [`into_result`](Self::into_result) or `.await`). pub fn is_ok(&self) -> bool { - matches!(&self.0, ResponseState::Ready(Ok(_))) + self.ready_ref("is_ok").is_ok() } - /// Returns `true` if the response contains `Err`. - /// Only meaningful for ready responses. - /// Returns `false` for pending (`Receiver`) or consumed (`Done`) states. + /// Returns `true` if the response is ready and `Err`. Panics on a + /// pending response (see [`is_ok`](Self::is_ok)). pub fn is_err(&self) -> bool { - matches!(&self.0, ResponseState::Ready(Err(_))) + self.ready_ref("is_err").is_err() } - /// Maps the inner value if the response is ready and `Ok`. - /// - /// Panics on pending or consumed responses. - pub fn map U>(self, f: F) -> Response { - match self.0 { - ResponseState::Ready(result) => Response( - ResponseState::Ready(result.map(f)), - #[cfg(feature = "metrics")] - None, - ), + fn ready_ref(&self, op: &str) -> &Result { + match &self.0 { + ResponseState::Ready(result) => result, #[cfg(feature = "async")] - ResponseState::Receiver(_) | ResponseState::TimedReceiver(_) => { - panic!("called map() on a pending Response; use .await in async contexts") + ResponseState::Sent(_) | ResponseState::Receiver(_) => { + panic!("called {op}() on a pending Response; .await it, or use into_result()") } - ResponseState::Done => panic!("Response already consumed"), + ResponseState::Done => panic!("called {op}() on a Response already consumed"), } } -} -#[cfg(feature = "async")] -impl Response { - /// Create a `Response` from a oneshot receiver with a timeout. - /// - /// Used by tasks-mode protocol blanket impls. Returns - /// `Err(ActorError::RequestTimeout)` if the timeout expires. - pub fn from_with_timeout( - result: Result, ActorError>, - duration: Duration, - ) -> Self { - #[cfg(feature = "metrics")] - let observation = crate::instrumentation::RequestObservation::new(); - match result { - Ok(rx) => { - let fut = Box::pin(async move { - match spawned_rt::tasks::timeout(duration, rx).await { - Ok(Ok(val)) => Ok(val), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - } - }); - Self( - ResponseState::TimedReceiver(fut), - #[cfg(feature = "metrics")] - Some(observation), - ) - } - Err(e) => { - let result = Err(e); - #[cfg(feature = "metrics")] - observation.finish(&result); - Self( - ResponseState::Ready(result), - #[cfg(feature = "metrics")] - None, - ) - } - } + /// Maps the inner value. A pending response is waited for (see + /// [`Response`]). + pub fn map U>(self, f: F) -> Response { + Response::ready(self.resolve("map").map(f)) } } #[cfg(feature = "async")] -impl From, ActorError>> for Response { - fn from(result: Result, ActorError>) -> Self { +impl From, ActorError>> for Response { + fn from(result: Result, ActorError>) -> Self { #[cfg(feature = "metrics")] let observation = crate::instrumentation::RequestObservation::new(); match result { Ok(rx) => Self( - ResponseState::Receiver(rx), + ResponseState::Sent(rx), #[cfg(feature = "metrics")] Some(observation), ), @@ -219,21 +219,15 @@ impl Future for Response { _cx: &mut std::task::Context<'_>, ) -> std::task::Poll { let this = self.get_mut(); + #[cfg(feature = "async")] + if matches!(this.0, ResponseState::Sent(_)) { + if let ResponseState::Sent(rx) = std::mem::replace(&mut this.0, ResponseState::Done) { + this.0 = ResponseState::Receiver(Box::pin(rx.into_timed_future())); + } + } let result = match &mut this.0 { #[cfg(feature = "async")] - ResponseState::Receiver(rx) => match Pin::new(rx).poll(_cx) { - std::task::Poll::Ready(Ok(val)) => { - this.0 = ResponseState::Done; - std::task::Poll::Ready(Ok(val)) - } - std::task::Poll::Ready(Err(_)) => { - this.0 = ResponseState::Done; - std::task::Poll::Ready(Err(ActorError::ActorStopped)) - } - std::task::Poll::Pending => std::task::Poll::Pending, - }, - #[cfg(feature = "async")] - ResponseState::TimedReceiver(fut) => match fut.as_mut().poll(_cx) { + ResponseState::Receiver(rx) => match rx.as_mut().poll(_cx) { std::task::Poll::Ready(result) => { this.0 = ResponseState::Done; std::task::Poll::Ready(result) @@ -244,6 +238,8 @@ impl Future for Response { ResponseState::Ready(result) => std::task::Poll::Ready(result), _ => unreachable!(), }, + #[cfg(feature = "async")] + ResponseState::Sent(_) => unreachable!("converted above"), ResponseState::Done => panic!("Response polled after completion"), }; #[cfg(feature = "metrics")] @@ -260,7 +256,7 @@ impl Future for Response { mod tests { use super::*; #[cfg(feature = "async")] - use spawned_rt::tasks::oneshot; + use crate::reply::Deadline; #[test] fn ready_ok_unwrap() { @@ -270,71 +266,135 @@ mod tests { #[test] fn ready_err_is_err() { - let r: Response = Response::ready(Err(ActorError::ActorStopped)); + let r: Response = Response::ready(Err(ActorError::ActorStopped( + crate::error::ExitReason::Normal, + ))); assert!(r.is_err()); } #[test] #[should_panic(expected = "ActorStopped")] fn ready_err_unwrap_panics() { - let r: Response = Response::ready(Err(ActorError::ActorStopped)); + let r: Response = Response::ready(Err(ActorError::ActorStopped( + crate::error::ExitReason::Normal, + ))); r.unwrap(); } #[test] #[cfg(feature = "async")] fn future_resolves_from_receiver() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (tx, rx) = oneshot::channel::(); - let resp: Response = Response::from(Ok(rx)); - tx.send(99).unwrap(); - let val = resp.await.unwrap(); - assert_eq!(val, 99); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::future_resolves_from_receiver"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let resp: Response = Response::from(Ok(rx)); + reply.reply(99); + assert_eq!(resp.await.unwrap(), 99); + }); + }, + ); } #[test] #[cfg(feature = "async")] - fn future_err_on_dropped_sender() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (tx, rx) = oneshot::channel::(); - let resp: Response = Response::from(Ok(rx)); - drop(tx); - let result = resp.await; - assert!(matches!(result, Err(ActorError::ActorStopped))); - }); + fn future_err_on_dropped_reply() { + crate::test_wait::bounded( + concat!(module_path!(), "::future_err_on_dropped_reply"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let resp: Response = Response::from(Ok(rx)); + drop(reply); + assert!(matches!(resp.await, Err(ActorError::ReplyDropped))); + }); + }, + ); } + /// Review footgun (tasks): `actor.query().unwrap()` used to panic on a + /// pending response and `is_ok()` answered `false`. Off a runtime it now + /// waits for the reply; inside one it panics, and `is_ok` never answers + /// `false` for a reply it has not seen. #[test] - fn map_transforms_value() { - let r: Response = Response::ready(Ok(2)); - let mapped = r.map(|x| x * 3); - assert_eq!(mapped.unwrap(), 6); + #[cfg(all(feature = "async", not(spawned_sim)))] + fn pending_response_waits_off_a_runtime() { + crate::test_wait::bounded( + concat!(module_path!(), "::pending_response_waits_off_a_runtime"), + move || { + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let answer = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + reply.reply(7); + }); + let resp: Response = Response::from(Ok(rx)); + assert_eq!(resp.unwrap(), 7); + answer.join().unwrap(); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + drop(reply); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.into_result(), Err(ActorError::ReplyDropped))); + + let (reply, rx) = crate::reply::reply_pair::(Deadline::Never); + reply.reply(2); + assert_eq!(Response::from(Ok(rx)).map(|x| x * 5).unwrap(), 10); + }, + ); + } + + #[test] + #[cfg(all(feature = "async", not(spawned_sim)))] + #[should_panic(expected = "inside an async runtime")] + fn pending_response_unwrap_panics_inside_a_runtime() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::pending_response_unwrap_panics_inside_a_runtime" + ), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); + Response::from(Ok(rx)).unwrap(); + }); + }, + ); } #[test] #[cfg(feature = "async")] - fn timed_receiver_resolves() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (tx, rx) = oneshot::channel::(); - let resp = Response::from_with_timeout(Ok(rx), Duration::from_secs(5)); - tx.send(42).unwrap(); - assert_eq!(resp.await.unwrap(), 42); - }); + #[should_panic(expected = "pending Response")] + fn is_ok_on_a_pending_response_panics_instead_of_answering_false() { + let (_reply, rx) = crate::reply::reply_pair::(Deadline::Never); + let _ = Response::from(Ok(rx)).is_ok(); + } + + #[test] + fn map_transforms_value() { + let r: Response = Response::ready(Ok(2)); + let mapped = r.map(|x| x * 3); + assert_eq!(mapped.unwrap(), 6); } #[test] #[cfg(feature = "async")] - fn timed_receiver_times_out() { - let rt = spawned_rt::tasks::Runtime::new().unwrap(); - rt.block_on(async { - let (_tx, rx) = oneshot::channel::(); - let resp = Response::from_with_timeout(Ok(rx), Duration::from_millis(50)); - let result = resp.await; - assert!(matches!(result, Err(ActorError::RequestTimeout))); - }); + fn receiver_times_out_at_its_send_deadline() { + crate::test_wait::bounded( + concat!(module_path!(), "::receiver_times_out_at_its_send_deadline"), + move || { + let rt = spawned_rt::tasks::Runtime::new().unwrap(); + rt.block_on(async { + let (reply, rx) = + crate::reply::reply_pair::(Deadline::after(Duration::from_millis(50))); + let resp: Response = Response::from(Ok(rx)); + assert!(matches!(resp.await, Err(ActorError::RequestTimeout))); + assert_eq!(reply.reply(1), crate::reply::Delivery::Expired); + }); + }, + ); } } diff --git a/concurrency/src/supervision.rs b/concurrency/src/supervision.rs index 682d71a2d..da460669b 100644 --- a/concurrency/src/supervision.rs +++ b/concurrency/src/supervision.rs @@ -38,16 +38,35 @@ pub enum RestartMode { Temporary, } impl RestartMode { - fn restarts(self, reason: Option<&ExitReason>) -> bool { + /// The restart table. A Transient child restarts if its final reason is + /// abnormal or a cleanup failure (a destructor panic after its death + /// point) was recorded: `is_abnormal()` alone no longer predicts it. + fn restarts(self, reason: Option<&ExitReason>, cleanup_failed: bool) -> bool { match self { Self::Permanent => true, - Self::Transient => reason.map(ExitReason::is_abnormal).unwrap_or(true), + Self::Transient => { + cleanup_failed || reason.map(ExitReason::is_abnormal).unwrap_or(true) + } Self::Temporary => false, } } } +/// Trace projection of a child's exit: the raw reason class, or the abnormal +/// class when a cleanup failure was recorded, so `Supervision.tla`'s restart +/// decision (`RestartFollowsPolicy`) stays valid. +fn exit_class(reason: &ExitReason, cleanup_failed: bool) -> &'static str { + if cleanup_failed && !reason.is_abnormal() { + "Panic" + } else { + trace::reason(Some(reason)) + } +} + +/// Restart delay policy. Non-exhaustive: start from `Backoff::default()` (or +/// `new()`) and chain the setters, or assign the public fields of a value. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct Backoff { /// First retry's delay cap, not its minimum. With jitter enabled the delay /// can be anywhere from zero through this value (1s permits a 150ms retry). @@ -77,6 +96,37 @@ impl Default for Backoff { } } impl Backoff { + /// The default policy: 10 ms initial cap, 5 s maximum, stable after + /// 30 s, jitter on, no fixed seed. + pub fn new() -> Self { + Self::default() + } + /// Set [`initial`](Self::initial). + pub fn initial(mut self, initial: Duration) -> Self { + self.initial = initial; + self + } + /// Set [`maximum`](Self::maximum). + pub fn maximum(mut self, maximum: Duration) -> Self { + self.maximum = maximum; + self + } + /// Set [`stable_after`](Self::stable_after). + pub fn stable_after(mut self, stable_after: Duration) -> Self { + self.stable_after = stable_after; + self + } + /// Set [`jitter`](Self::jitter). + pub fn jitter(mut self, jitter: bool) -> Self { + self.jitter = jitter; + self + } + /// Seed the jitter for reproducible tests. + pub fn seed(mut self, seed: u64) -> Self { + self.seed = Some(seed); + self + } + /// Capped exponential delay before jitter. pub(crate) fn bound(&self, failures: u32) -> Duration { self.initial @@ -121,6 +171,7 @@ pub(crate) fn jitter_seed(seed: u64, name: &[u8]) -> u64 { /// The affected children are fully joined before ordered reconstruction. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[non_exhaustive] pub enum SupervisorStrategy { #[default] OneForOne, @@ -128,12 +179,20 @@ pub enum SupervisorStrategy { RestForOne, } +/// Supervisor flags (OTP's `strategy`, `intensity` and `period`, plus a +/// child cap and a restart backoff). Non-exhaustive: start from +/// `SupervisorConfig::default()` (or `new()`) and chain the setters, +/// `SupervisorConfig::new().strategy(SupervisorStrategy::OneForAll).max_restarts(3)`, +/// or assign the public fields of a value. #[derive(Debug, Clone)] +#[non_exhaustive] pub struct SupervisorConfig { pub strategy: SupervisorStrategy, - /// Supervisor-wide child retry grants per window, including collateral - /// OneForAll/RestForOne replacements. A six-child OneForAll round requires - /// six available grants. This retains v0.7's per-child accounting. + /// Supervisor-wide restarts per `restart_window` (OTP's intensity and + /// period). One restart is one restart action: a OneForOne child retry, + /// or one OneForAll/RestForOne recovery round, however many children + /// the round rebuilds. A failed replacement inside a round requests a + /// new round, which is charged again (OTP's `try_again_restart`). pub max_restarts: usize, pub restart_window: Duration, pub max_children: usize, @@ -150,8 +209,40 @@ impl Default for SupervisorConfig { } } } +impl SupervisorConfig { + /// OneForOne, 5 restarts per 30 s, 1024 children, the default backoff. + pub fn new() -> Self { + Self::default() + } + /// Set [`strategy`](Self::strategy). + pub fn strategy(mut self, strategy: SupervisorStrategy) -> Self { + self.strategy = strategy; + self + } + /// Set [`max_restarts`](Self::max_restarts) (OTP's intensity). + pub fn max_restarts(mut self, max_restarts: usize) -> Self { + self.max_restarts = max_restarts; + self + } + /// Set [`restart_window`](Self::restart_window) (OTP's period). + pub fn restart_window(mut self, restart_window: Duration) -> Self { + self.restart_window = restart_window; + self + } + /// Set [`max_children`](Self::max_children). + pub fn max_children(mut self, max_children: usize) -> Self { + self.max_children = max_children; + self + } + /// Set [`backoff`](Self::backoff). + pub fn backoff(mut self, backoff: Backoff) -> Self { + self.backoff = backoff; + self + } +} #[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] pub enum SupervisorError { #[error(transparent)] System(#[from] SystemError), @@ -161,13 +252,33 @@ pub enum SupervisorError { DuplicateChild(String), #[error("child capacity exceeded")] Capacity, + /// The start was cancelled or the system closed before it completed. #[error("supervisor stopped before startup completed")] Stopped, #[error("supervisor failed: {0}")] Failed(String), + /// A child's first start failed (S1, OTP `failed_to_start_child`): its + /// factory errored or panicked, it stopped or panicked before readiness, + /// it missed `startup_timeout`, or its placement or group failed. + /// Returned after the rollback finished; `rollback_failures` lists + /// cleanup failures of that rollback. + #[error("failed to start child {child}: {cause}")] + FailedToStartChild { + child: String, + cause: String, + rollback_failures: Vec, + }, + /// A child's cleanup got stuck during the initial start (S2). + #[error("child cleanup stuck during initial start: {child}: {cause}")] + StartupCleanupStuck { + child: String, + cause: String, + rollback_failures: Vec, + }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum ChildState { Waiting, Starting, @@ -179,6 +290,7 @@ pub enum ChildState { } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct ChildSnapshot { pub name: String, pub state: ChildState, @@ -234,6 +346,7 @@ pub struct ChildSpec { restart: RestartMode, startup_timeout: Duration, shutdown_timeout: Duration, + bounds: crate::reply::jobs::Bounds, } impl ChildSpec { /// Create a permanent child with five-second startup and shutdown limits. @@ -255,8 +368,22 @@ impl ChildSpec { restart: RestartMode::Permanent, startup_timeout: Duration::from_secs(5), shutdown_timeout: Duration::from_secs(5), + bounds: Default::default(), } } + /// Override `Actor::MAX_OUTSTANDING` (deferred credits per instance, + /// capped at 65535) for every generation of this child. + pub fn max_outstanding(mut self, max: usize) -> Self { + self.bounds.max_outstanding = Some(max); + self + } + /// Override `Actor::MAX_NATIVE_JOBS` (blocking jobs running at once) for + /// every generation of this child. The default (at most 4) is easy to + /// outgrow; its only symptom under load is `NativeBudget` refusals. + pub fn max_native_jobs(mut self, max: usize) -> Self { + self.bounds.max_native_jobs = Some(max); + self + } pub fn restart(mut self, restart: RestartMode) -> Self { self.restart = restart; self @@ -405,6 +532,29 @@ impl View { self.done.send_replace(true); true } + /// Cleanup stuck past its deadlines: the snapshot settles as Failed, + /// which releases a waiting `start()` (S2), while the controller keeps + /// awaiting the real join. + fn mark_stuck(&self) { + self.change(|data| { + data.current = None; + data.snapshot.state = ChildState::Failed; + }); + } + /// Terminal, or settled as Failed by a stuck cleanup. + async fn settled(&self) { + let mut changes = self.changed.subscribe(); + loop { + changes.borrow_and_update(); + let state = system::lock(&self.data).snapshot.state; + if matches!(state, ChildState::Stopped | ChildState::Failed) || *self.done.borrow() { + return; + } + if changes.changed().await.is_err() { + return; + } + } + } fn finish(&self, failed: bool) { self.change(|data| { data.current = None; @@ -486,6 +636,7 @@ trait ChildControl: Send + Sync { fn initial(&self) -> watch::Receiver; fn shutdown_timeout(&self) -> Duration; fn finish_unstarted(&self, failed: bool); + fn settled(&self) -> BoxFuture<'_, ()>; } impl ChildControl for View { fn assign_recovery(&self, slot: Arc) -> bool { @@ -509,11 +660,13 @@ impl ChildControl for View { self.changed.send_modify(|n| *n = n.wrapping_add(1)); true } + /// Collateral stop (OneForAll/RestForOne): the supervisor's parent exit + /// signal (E3); the controller escalates to a kill at `shutdown_timeout`. fn stop_generation(&self, slot: &RecoverySlot) { slot.stop.cancel(); let scope = system::lock(&self.data).scope.clone(); if let Some(scope) = scope { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn snapshot(&self) -> ChildSnapshot { @@ -532,10 +685,15 @@ impl ChildControl for View { } self.changed.send_modify(|n| *n = n.wrapping_add(1)); } + /// Supervisor shutdown of this child (V9, reverse order): the parent + /// exit signal `exit(Child, shutdown)` (E3). A non-trapping child ends + /// `Shutdown` without `stopped()`; a trapping child finishes its earlier + /// messages and runs `stopped()`. The controller kills it at + /// `shutdown_timeout`. fn stop(&self) { self.stop_turn.cancel(); if let Some(scope) = system::lock(&self.data).scope.clone() { - scope.stop(ExitReason::Shutdown); + scope.signal_parent(ExitReason::Shutdown); } } fn done(&self) -> watch::Receiver { @@ -552,6 +710,9 @@ impl ChildControl for View { self.finish(failed); } } + fn settled(&self) -> BoxFuture<'_, ()> { + Box::pin(View::settled(self)) + } } struct RetryBudget { @@ -603,6 +764,8 @@ pub(crate) fn admit_in_window( #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum FailureKind { Other, + /// A child's first start failed (S1). + FirstStart, Cleanup, Coordinator, RestartBudget, @@ -649,6 +812,9 @@ struct Core { failure: Mutex>, failed: Arc, cleanup_failure: Mutex>, + /// Every recorded cleanup failure, for the rollback list of a failed + /// start (S2). + cleanup_failures: Mutex>, children: Mutex>>, budget: Mutex, config: SupervisorConfig, @@ -750,6 +916,7 @@ impl Core { let _ = make; } fn record_cleanup_failure(&self, error: &str) { + system::lock(&self.cleanup_failures).push(error.to_string()); let mut failure = system::lock(&self.cleanup_failure); if failure.is_none() { *failure = Some(error.to_string()); @@ -897,6 +1064,15 @@ impl Core { .flatten() }) } + /// One OneForAll/RestForOne recovery round is one restart action (OTP's + /// `add_restart` runs once per restart, before the cohort is rebuilt): + /// take its grant now. The round's controllers then rebuild without a + /// grant of their own. Pending per-child reservations (retries during + /// the initial start) keep their capacity. Returns whether it was taken. + fn charge_round(&self) -> bool { + let mut budget = system::lock(&self.budget); + budget.take(Instant::now()) + } /// Caller holds the admission gate and slot lock; charge atomically with /// generation admission, so placement failure or supersession refunds it. fn retry_under_gate(&self, reservation: &mut RetryReservation<'_>) -> bool { @@ -1047,6 +1223,7 @@ impl SupervisorBuilder { failure: Mutex::new(None), failed: Arc::new(std::sync::atomic::AtomicBool::new(false)), cleanup_failure: Mutex::new(None), + cleanup_failures: Mutex::new(Vec::new()), children: Mutex::new(Vec::new()), budget: Mutex::new(RetryBudget { attempts: VecDeque::new(), @@ -1199,16 +1376,52 @@ impl SupervisorBuilder { // Readiness is historical, just like ActorRef::ready(). A later system // admission close must not turn successful startup into rollback. let initialized = *core.ready.borrow(); - let result = if initialized { + if initialized { guard.armed = false; - Ok(Supervisor { core: core.clone() }) - } else { - Err(system::lock(&core.failure) - .clone() - .map(|failure| SupervisorError::Failed(failure.message)) - .unwrap_or(SupervisorError::Stopped)) + return Ok(Supervisor { core: core.clone() }); + } + // S2: the error is returned only after the rollback finished: every + // child is terminal, or settled as Failed by its stuck cleanup. + let children = system::lock(&core.children).clone(); + for child in &children { + child.settled().await; + } + Err(core.start_error()) + } +} +impl Core { + /// The cause of a failed initial start (S2), with the rollback's cleanup + /// failures listed separately. + fn start_error(&self) -> SupervisorError { + let Some(failure) = system::lock(&self.failure).clone() else { + return SupervisorError::Stopped; }; - result + let rollback_failures: Vec = system::lock(&self.cleanup_failures) + .iter() + .filter(|f| **f != failure.message) + .cloned() + .collect(); + match (failure.kind, failure.origin) { + (FailureKind::FirstStart, Some(origin)) => { + let prefix = format!("failed to start child {}: ", origin.child); + let cause = failure + .message + .strip_prefix(&prefix) + .unwrap_or(&failure.message) + .to_string(); + SupervisorError::FailedToStartChild { + child: origin.child, + cause, + rollback_failures, + } + } + (FailureKind::Cleanup, Some(origin)) => SupervisorError::StartupCleanupStuck { + child: origin.child, + cause: failure.message, + rollback_failures, + }, + _ => SupervisorError::Failed(failure.message), + } } } impl Drop for SupervisorBuilder { @@ -1243,6 +1456,7 @@ pub struct Supervisor { core: Arc, } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct SupervisorReport { pub complete: bool, pub failure: Option, @@ -1272,9 +1486,9 @@ impl Supervisor { pub async fn join(&self) { let _ = system::wait_true(self.core.done.subscribe()).await; } - pub async fn shutdown(&self, deadline: Duration) -> SupervisorReport { + pub async fn shutdown(&self, grace: Duration) -> SupervisorReport { self.stop(); - let complete = rt::timeout(deadline, system::wait_true(self.core.done.subscribe())) + let complete = rt::timeout(grace, system::wait_true(self.core.done.subscribe())) .await .unwrap_or(false); if !complete { @@ -1503,6 +1717,12 @@ async fn run_recoveries(core: Arc) { } let round = trace::atomic(|| { let mut queue = system::lock(&core.recovery); + // The round is the restart action: charge it once, before any + // child of the cohort is stopped or rebuilt. + let origin = queue.pending.iter().position(|pending| *pending)?; + if !core.charge_round() { + return Some(Err(origin)); + } queue .select(core.config.strategy == SupervisorStrategy::OneForAll) .map(|(origin, start, id)| { @@ -1516,10 +1736,21 @@ async fn run_recoveries(core: Arc) { .round(id) .count(start + 1) }); - (start, meta) + Ok((start, meta)) }) }); - let Some((start, meta)) = round else { + if let Some(Err(origin)) = round { + // Exhausted: a concurrent stop wins; only an open supervisor fails. + let snapshot = children[origin].snapshot(); + let message = format!("restart budget exhausted by {}", snapshot.name); + core.fail_record_if_open_as( + snapshot_failure(snapshot, FailureKind::RestartBudget, message), + || None, + ); + core.log_failure_once(); + return; + } + let Some(Ok((start, meta))) = round else { if matches!( select( Box::pin(core.admission_closed.cancelled()), @@ -1641,6 +1872,8 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc { if !stop_turn { + let failed = child.cleanup_failure().is_some(); trace::emit(core.trace, || { - Event::new("SelfExit", c, trace::reason(Some(&reason))) + Event::new("SelfExit", c, exit_class(&reason, failed)) }); } reason } + // E3: the supervisor stopped this child and it + // did not reach its death point within + // shutdown_timeout: the parent's kill, then a + // second interval before cleanup is stuck. + Err(_) if stop_turn => { + scope.kill(); + // The second interval runs from the kill. + cleanup_until = Some(Instant::now()); + match rt::timeout(spec.shutdown_timeout, child.wait_exit_async()) + .await + { + Ok(reason) => reason, + Err(_) => { + stuck = true; + // Reply traces: the cleanup deadline + // passed before the child's death + // point (DeferredReplies `CleanupDeadline`). + crate::lifecycle_trace::reply( + child.id().raw(), + "Stuck", + 0, + "", + ); + core.fail_cleanup_for( + view.snapshot(), + format!("actor cleanup stuck for {}", spec.name), + || Some(Event::new("CleanupStuck", c, "")), + ); + view.mark_stuck(); + child.wait_exit_async().await + } + } + } Err(_) => { if !stop_turn { trace::emit(core.trace, || { @@ -2062,11 +2345,13 @@ async fn run_child(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(spec: ChildSpec, view: Arc>, core: Arc(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - first.view.finish(true); - first.view.finish_unstarted(false); // stale non-failed sweep races last - second.view.finish_unstarted(false); - second.view.finish(true); // failed controller races last - assert_eq!(first.snapshot().state, ChildState::Failed); - assert_eq!(second.snapshot().state, ChildState::Failed); - tree.core - .fail("restart budget exhausted by a user-provided message".into()); - assert_eq!( - system::lock(&tree.core.failure).as_ref().unwrap().kind, - FailureKind::Other - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::failure_classification_is_typed_and_terminal_failure_is_sticky" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("terminal-order", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + first.view.finish(true); + first.view.finish_unstarted(false); // stale non-failed sweep races last + second.view.finish_unstarted(false); + second.view.finish(true); // failed controller races last + assert_eq!(first.snapshot().state, ChildState::Failed); + assert_eq!(second.snapshot().state, ChildState::Failed); + tree.core + .fail("restart budget exhausted by a user-provided message".into()); + assert_eq!( + system::lock(&tree.core.failure).as_ref().unwrap().kind, + FailureKind::Other + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; } #[tokio::test] async fn initial_placement_failure_has_deterministic_failed_slots() { - use crate::tasks::ActorStart; - for _ in 0..8 { - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("placement-failure", 1).unwrap(); - let occupied = CoordinatorWorker.start_on(&group).unwrap(); - rt::timeout(Duration::from_secs(10), occupied.ready()) - .await - .unwrap() - .unwrap(); - let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); - let first = tree - .add(ChildSpec::new("first", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let second = tree - .add(ChildSpec::new("second", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = tree.core.clone(); - assert!(tree.start().await.is_err()); - assert!(rt::timeout( - Duration::from_secs(10), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - for child in [first, second] { - assert_eq!(child.snapshot().generation, 0); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(matches!( - child.current(), - Err(ChildUnavailable(ChildState::Failed)) - )); - } - assert!(system.shutdown(Duration::from_secs(10)).await.complete); - } + crate::test_wait::within( + concat!( + module_path!(), + "::initial_placement_failure_has_deterministic_failed_slots" + ), + async move { + use crate::tasks::ActorStart; + for _ in 0..8 { + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("placement-failure", 1).unwrap(); + let occupied = CoordinatorWorker.start_on(&group).unwrap(); + rt::timeout(Duration::from_secs(10), occupied.ready()) + .await + .unwrap() + .unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let first = tree + .add(ChildSpec::new("first", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let second = tree + .add(ChildSpec::new("second", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = tree.core.clone(); + assert!(tree.start().await.is_err()); + assert!(rt::timeout( + Duration::from_secs(10), + system::wait_true(core.done.subscribe()) + ) + .await + .unwrap()); + for child in [first, second] { + assert_eq!(child.snapshot().generation, 0); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(matches!( + child.current(), + Err(ChildUnavailable(ChildState::Failed)) + )); + } + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + } + }, + ) + .await; } struct PanicRunner; impl Runner for PanicRunner { @@ -2405,6 +2736,7 @@ mod policy_tests { impl Actor for CoordinatorWorker {} #[tokio::test] async fn coordinator_fault_joins_launched_controllers_and_helpers() { + crate::test_wait::within(concat!(module_path!(), "::coordinator_fault_joins_launched_controllers_and_helpers"), async move { let system = ActorSystem::new().unwrap(); let group = system .async_group("supervisor-coordinator-fault", 1) @@ -2432,9 +2764,9 @@ mod policy_tests { .unwrap(); tree.runners[1] = Box::new(PanicRunner); let core = tree.core.clone(); - assert!( - matches!(tree.start().await, Err(SupervisorError::Failed(e)) if e.contains("injected supervisor coordinator fault")) - ); + // v0.8 (S2): start() returns its error only after the rollback, which + // waits for the live child's owned helper. + let start = rt::spawn(tree.start()); assert!(rt::timeout( Duration::from_millis(30), system::wait_true(core.done.subscribe()) @@ -2442,6 +2774,9 @@ mod policy_tests { .await .is_err()); release.send(()).unwrap(); + assert!( + matches!(start.await.unwrap(), Err(SupervisorError::Failed(e)) if e.contains("injected supervisor coordinator fault")) + ); assert!(rt::timeout( Duration::from_secs(10), system::wait_true(core.done.subscribe()) @@ -2457,176 +2792,136 @@ mod policy_tests { .supervisor_failures .iter() .any(|e| e.contains("injected supervisor coordinator fault"))); + }).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stop_wins_over_late_child_failure_without_trace_serialization() { - let system = ActorSystem::new().unwrap(); - let tree = system.supervisor(SupervisorConfig::default()).unwrap(); - tree.core.close(); - assert!(!tree - .core - .fail_if_open_as("late child failure".into(), || None)); - assert!(system::lock(&tree.core.failure).is_none()); - assert!(tree - .core - .gate - .if_open(|| tree.core.ready.send_replace(true)) - .is_none()); - assert!(!*tree.core.ready.borrow()); - // Cleanup failures remain observable after ordinary admission closure. - tree.core.fail_cleanup("late cleanup failure".into()); - assert_eq!( - system::lock(&tree.core.failure).as_deref(), - Some("late cleanup failure") - ); - drop(tree); - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::stop_wins_over_late_child_failure_without_trace_serialization" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system.supervisor(SupervisorConfig::default()).unwrap(); + tree.core.close(); + assert!(!tree + .core + .fail_if_open_as("late child failure".into(), || None)); + assert!(system::lock(&tree.core.failure).is_none()); + assert!(tree + .core + .gate + .if_open(|| tree.core.ready.send_replace(true)) + .is_none()); + assert!(!*tree.core.ready.borrow()); + // Cleanup failures remain observable after ordinary admission closure. + tree.core.fail_cleanup("late cleanup failure".into()); + assert_eq!( + system::lock(&tree.core.failure).as_deref(), + Some("late cleanup failure") + ); + drop(tree); + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, + ) + .await; } #[test] fn delayed_failure_notifications_are_absorbed_by_the_already_assigned_round() { - let mut queue = RecoveryQueue { - pending: vec![false; 3], - covered: vec![0; 3], - next_id: 1, - }; - assert!(queue.request(1, 0)); - queue.pending.fill(false); - // The coordinator publishes coverage before per-child assignment. - queue.covered[1..].fill(1); - assert!(!queue.request(2, 0)); - assert!(!queue.pending.iter().any(|value| *value)); - // A new failure in a reconstructed generation must still be admitted. - assert!(queue.request(2, 1)); - assert!(queue.request(0, 0)); // unaffected upstream in RestForOne + crate::test_wait::bounded( + concat!( + module_path!(), + "::delayed_failure_notifications_are_absorbed_by_the_already_assigned_round" + ), + move || { + let mut queue = RecoveryQueue { + pending: vec![false; 3], + covered: vec![0; 3], + next_id: 1, + }; + assert!(queue.request(1, 0)); + queue.pending.fill(false); + // The coordinator publishes coverage before per-child assignment. + queue.covered[1..].fill(1); + assert!(!queue.request(2, 0)); + assert!(!queue.pending.iter().any(|value| *value)); + // A new failure in a reconstructed generation must still be admitted. + assert!(queue.request(2, 1)); + assert!(queue.request(0, 0)); // unaffected upstream in RestForOne + }, + ); } #[tokio::test] async fn admission_closure_after_readiness_preserves_fixed_system_order() { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); + crate::test_wait::within( + concat!( + module_path!(), + "::admission_closure_after_readiness_preserves_fixed_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, } - if let Some(release) = self.release.take() { - let _ = release.await; + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; + } + } } - } - } - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { - Ordered { - id, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - async move { - Ok::<_, String>(Ordered { - id: 100, - log, + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), entered: None, release: None, - }) - } - })) - .unwrap(); - let resume = CancellationToken::new(); - let (passed, after_check) = watch::channel(false); - *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); - let tree = builder.start().await.unwrap(); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // The later standalone actor owns the current stop turn. Resume the - // supervisor exactly between readiness and its cancellation wait. - resume.cancel(); - assert!(rt::timeout(limit, system::wait_true(after_check)) - .await - .unwrap()); - assert!( - !tree.core.cancel.is_cancelled(), - "admission closure stole the supervisor's stop turn" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - } - - #[tokio::test] - async fn startup_completion_or_drop_after_readiness_preserves_system_order() { - use crate::tasks::ActorStart; - struct Ordered { - id: u32, - log: Arc>>, - entered: Option>, - release: Option>, - } - impl Actor for Ordered { - async fn stopped(&mut self, _: &crate::tasks::Context) { - system::lock(&self.log).push(self.id); - if let Some(entered) = self.entered.take() { - let _ = entered.send(()); - } - if let Some(release) = self.release.take() { - let _ = release.await; + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); } - } - } - for complete_start in [true, false] { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); - let log = Arc::new(Mutex::new(Vec::new())); - for id in 0..2 { + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + async move { + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let resume = CancellationToken::new(); + let (passed, after_check) = watch::channel(false); + *system::lock(&builder.core.initial_ready_pause) = Some((resume.clone(), passed)); + let tree = builder.start().await.unwrap(); Ordered { - id, + id: 200, log: log.clone(), entered: None, release: None, @@ -2636,412 +2931,578 @@ mod policy_tests { .ready() .await .unwrap(); - } - let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); - let child_log = log.clone(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - builder - .add(ChildSpec::new("middle", &group, move |_| { - let log = child_log.clone(); - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // The later standalone actor owns the current stop turn. Resume the + // supervisor exactly between readiness and its cancellation wait. + resume.cancel(); + assert!(rt::timeout(limit, system::wait_true(after_check)) + .await + .unwrap()); + assert!( + !tree.core.cancel.is_cancelled(), + "admission closure stole the supervisor's stop turn" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + }, + ) + .await; + } + + #[tokio::test] + async fn startup_completion_or_drop_after_readiness_preserves_system_order() { + crate::test_wait::within( + concat!( + module_path!(), + "::startup_completion_or_drop_after_readiness_preserves_system_order" + ), + async move { + use crate::tasks::ActorStart; + struct Ordered { + id: u32, + log: Arc>>, + entered: Option>, + release: Option>, + } + impl Actor for Ordered { + async fn started(&mut self, ctx: &crate::tasks::Context) { + // v0.8: stopped() records the order only for a trapping + // child (E3, decision 2). + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &crate::tasks::Context) { + system::lock(&self.log).push(self.id); + if let Some(entered) = self.entered.take() { + let _ = entered.send(()); + } + if let Some(release) = self.release.take() { + let _ = release.await; } - Ok::<_, String>(Ordered { - id: 100, - log, + } + } + for complete_start in [true, false] { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.thread_group("fixed-order-2-1-2", 5).unwrap(); + let log = Arc::new(Mutex::new(Vec::new())); + for id in 0..2 { + Ordered { + id, + log: log.clone(), entered: None, release: None, - }) + } + .start_on(&group) + .unwrap() + .ready() + .await + .unwrap(); } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_log = log.clone(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + builder + .add(ChildSpec::new("middle", &group, move |_| { + let log = child_log.clone(); + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Ordered { + id: 100, + log, + entered: None, + release: None, + }) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + Ordered { + id: 200, + log: log.clone(), + entered: None, + release: None, + } + .start_on(&group) + .unwrap() + .ready() .await + .unwrap(); + let (entered, arrival) = rt::oneshot::channel(); + let (release, gate) = rt::oneshot::channel(); + Ordered { + id: 201, + log: log.clone(), + entered: Some(entered), + release: Some(gate), + } + .start_on(&group) .unwrap() - ); - Ordered { - id: 200, - log: log.clone(), - entered: None, - release: None, - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let (entered, arrival) = rt::oneshot::channel(); - let (release, gate) = rt::oneshot::channel(); - Ordered { - id: 201, - log: log.clone(), - entered: Some(entered), - release: Some(gate), - } - .start_on(&group) - .unwrap() - .ready() - .await - .unwrap(); - let shutdown = system.shutdown(limit); - tokio::pin!(shutdown); - assert!(futures::poll!(&mut shutdown).is_pending()); - rt::timeout(limit, arrival).await.unwrap().unwrap(); - // Readiness committed, but the caller has not observed start's result. - // Admission now closes while the later actor still owns the stop turn. - if complete_start { - assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); - } else { - drop(startup); - } - assert!( - !core.cancel.is_cancelled(), - "completed readiness was rolled back" - ); - release.send(()).unwrap(); - assert!(rt::timeout(limit, shutdown).await.unwrap().complete); - assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); - } + .ready() + .await + .unwrap(); + let shutdown = system.shutdown(limit); + tokio::pin!(shutdown); + assert!(futures::poll!(&mut shutdown).is_pending()); + rt::timeout(limit, arrival).await.unwrap().unwrap(); + // Readiness committed, but the caller has not observed start's result. + // Admission now closes while the later actor still owns the stop turn. + if complete_start { + assert!(rt::timeout(limit, startup.as_mut()).await.unwrap().is_ok()); + } else { + drop(startup); + } + assert!( + !core.cancel.is_cancelled(), + "completed readiness was rolled back" + ); + release.send(()).unwrap(); + assert!(rt::timeout(limit, shutdown).await.unwrap().complete); + assert_eq!(*system::lock(&log), vec![201, 200, 100, 1, 0]); + } + }, + ) + .await; } #[tokio::test] async fn startup_success_remains_historical_after_unobserved_ready_then_failure() { - struct Idle; - impl Actor for Idle {} - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("historical-ready-failure", 1).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 0, - ..SupervisorConfig::default() - }) - .unwrap(); - // Hold construction until start() has been polled once, so the - // first poll is pending however fast the controller runs. - let (construct, gate) = rt::oneshot::channel::<()>(); - let gate = Arc::new(Mutex::new(Some(gate))); - let child = builder - .add(ChildSpec::new("ready-then-failed", &group, move |_| { - let gate = system::lock(&gate).take(); - async move { - if let Some(gate) = gate { - let _ = gate.await; - } - Ok::<_, String>(Idle) - } - })) - .unwrap(); - let core = builder.core.clone(); - let mut startup = Box::pin(builder.start()); - assert!(futures::poll!(startup.as_mut()).is_pending()); - construct.send(()).unwrap(); - assert!( - rt::timeout(limit, system::wait_true(core.ready.subscribe())) - .await - .unwrap() - ); - child - .current() - .unwrap() - .context() - .stop_with_reason(ExitReason::Kill); - assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) - .await - .unwrap()); - let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); - assert!(tree.failure().unwrap().contains("budget exhausted")); - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::startup_success_remains_historical_after_unobserved_ready_then_failure" + ), + async move { + struct Idle; + impl Actor for Idle {} + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("historical-ready-failure", 1).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 0, + ..SupervisorConfig::default() + }) + .unwrap(); + // Hold construction until start() has been polled once, so the + // first poll is pending however fast the controller runs. + let (construct, gate) = rt::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some(gate))); + let child = builder + .add(ChildSpec::new("ready-then-failed", &group, move |_| { + let gate = system::lock(&gate).take(); + async move { + if let Some(gate) = gate { + let _ = gate.await; + } + Ok::<_, String>(Idle) + } + })) + .unwrap(); + let core = builder.core.clone(); + let mut startup = Box::pin(builder.start()); + assert!(futures::poll!(startup.as_mut()).is_pending()); + construct.send(()).unwrap(); + assert!( + rt::timeout(limit, system::wait_true(core.ready.subscribe())) + .await + .unwrap() + ); + child + .current() + .unwrap() + .context() + .stop_with_reason(ExitReason::Kill); + assert!(rt::timeout(limit, system::wait_true(core.done.subscribe())) + .await + .unwrap()); + let tree = rt::timeout(limit, startup).await.unwrap().unwrap(); + assert!(tree.failure().unwrap().contains("budget exhausted")); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] async fn started_failures_allow_n_restarts_and_escalate_the_next_failure() { - use std::sync::atomic::{AtomicUsize, Ordering}; - struct FailsStarted(Arc); - impl Actor for FailsStarted { - async fn started(&mut self, _: &crate::tasks::Context) { - self.0.fetch_add(1, Ordering::SeqCst); - panic!("injected started failure"); - } - } - for allowed in [0, 1, 3] { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("started-budget", 1).unwrap(); - let count = Arc::new(AtomicUsize::new(0)); - let attempts = count.clone(); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: allowed, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("startup-failure", &group, move |_| { - let attempts = attempts.clone(); - async move { Ok::<_, String>(FailsStarted(attempts)) } - })) - .unwrap(); - let core = builder.core.clone(); - assert!(matches!( - rt::timeout(Duration::from_secs(30), builder.start()) + crate::test_wait::within( + concat!( + module_path!(), + "::started_failures_allow_n_restarts_and_escalate_the_next_failure" + ), + async move { + use std::sync::atomic::{AtomicUsize, Ordering}; + // v0.8 (S1): a first start is not retried, so the first generation + // is healthy and every replacement panics in started(). + struct FailsStarted(Arc, bool); + impl Actor for FailsStarted { + async fn started(&mut self, _: &crate::tasks::Context) { + if self.1 { + self.0.fetch_add(1, Ordering::SeqCst); + panic!("injected started failure"); + } + } + } + for allowed in [0, 1, 3] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("started-budget", 1).unwrap(); + let count = Arc::new(AtomicUsize::new(0)); + let attempts = count.clone(); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: allowed, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("startup-failure", &group, move |ctx| { + let attempts = attempts.clone(); + let fail = ctx.generation() > 1; + async move { Ok::<_, String>(FailsStarted(attempts, fail)) } + })) + .unwrap(); + let core = builder.core.clone(); + rt::timeout(Duration::from_secs(30), builder.start()) + .await + .unwrap() + .unwrap(); + child.current().unwrap().child_handle().kill(); + assert!(rt::timeout( + Duration::from_secs(30), + system::wait_true(core.done.subscribe()) + ) .await - .unwrap(), - Err(SupervisorError::Failed(_)) - )); - assert!(rt::timeout( - Duration::from_secs(30), - system::wait_true(core.done.subscribe()) - ) - .await - .unwrap()); - assert_eq!(count.load(Ordering::SeqCst), allowed + 1); - assert_eq!(child.snapshot().generation as usize, allowed + 1); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert_eq!(child.snapshot().state, ChildState::Failed); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); - } + .unwrap()); + assert_eq!(count.load(Ordering::SeqCst), allowed); + assert_eq!(child.snapshot().generation as usize, allowed + 1); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert_eq!(child.snapshot().state, ChildState::Failed); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + } + }, + ) + .await; } #[tokio::test] async fn sliding_window_expires_only_old_grants_before_the_next_retry() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("sliding-budget", 1).unwrap(); - let window = Duration::from_secs(3600); - let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 2, - restart_window: window, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let child = builder - .add(ChildSpec::new("worker", &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(); - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let mut actor = child.current().unwrap(); - for generation in 1..=2 { - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("retry".into())); - actor = rt::timeout( - Duration::from_secs(30), - child.wait_for_generation(generation), - ) - .await - .unwrap() - .unwrap(); - } - { - let mut budget = system::lock(&core.budget); - assert_eq!(budget.attempts.len(), 2); - // Control just the oldest timestamp; the real controller still owns - // admission and retry. The second grant remains inside the window. - budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); - } - actor - .child_handle() - .stop_with_reason(ExitReason::Panic("expired oldest".into())); - let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) - .await - .unwrap() - .unwrap(); - assert_eq!(system::lock(&core.budget).attempts.len(), 2); - fourth - .child_handle() - .stop_with_reason(ExitReason::Panic("now exhausted".into())); - rt::timeout(Duration::from_secs(30), tree.join()) - .await - .unwrap(); - assert_eq!(child.snapshot().generation, 4); - assert_eq!( - system::lock(&core.failure).as_ref().unwrap().kind, - FailureKind::RestartBudget - ); - assert!(system.shutdown(Duration::from_secs(30)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::sliding_window_expires_only_old_grants_before_the_next_retry" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sliding-budget", 1).unwrap(); + let window = Duration::from_secs(3600); + let mut builder = system + .supervisor(SupervisorConfig { + max_restarts: 2, + restart_window: window, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let child = builder + .add(ChildSpec::new("worker", &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(); + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let mut actor = child.current().unwrap(); + for generation in 1..=2 { + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("retry".into())); + actor = rt::timeout( + Duration::from_secs(30), + child.wait_for_generation(generation), + ) + .await + .unwrap() + .unwrap(); + } + { + let mut budget = system::lock(&core.budget); + assert_eq!(budget.attempts.len(), 2); + // Control just the oldest timestamp; the real controller still owns + // admission and retry. The second grant remains inside the window. + budget.attempts[0] = Instant::now().checked_sub(window).unwrap(); + } + actor + .child_handle() + .stop_with_reason(ExitReason::Panic("expired oldest".into())); + let fourth = rt::timeout(Duration::from_secs(30), child.wait_for_generation(3)) + .await + .unwrap() + .unwrap(); + assert_eq!(system::lock(&core.budget).attempts.len(), 2); + fourth + .child_handle() + .stop_with_reason(ExitReason::Panic("now exhausted".into())); + rt::timeout(Duration::from_secs(30), tree.join()) + .await + .unwrap(); + assert_eq!(child.snapshot().generation, 4); + assert_eq!( + system::lock(&core.failure).as_ref().unwrap().kind, + FailureKind::RestartBudget + ); + assert!(system.shutdown(Duration::from_secs(30)).await.complete); + }, + ) + .await; } + /// OTP charges `add_restart` when it decides a restart, before the + /// cohort is rebuilt: a superseded round keeps its charge, and the + /// superseding round is charged once more. #[tokio::test] - async fn supersession_before_generation_admission_refunds_the_retry() { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system - .async_group("refund-superseded-admission", 2) - .unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::RestForOne, - max_restarts: 3, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for name in ["upstream", "middle", "tail"] { - children.push( - builder - .add(ChildSpec::new(name, &group, |_| async { - Ok::<_, String>(CoordinatorWorker) - })) - .unwrap(), - ); - } - let core = builder.core.clone(); - let tree = builder.start().await.unwrap(); - let resume = CancellationToken::new(); - let (entered, arrival) = watch::channel(false); - *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); - children[1] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("middle fails".into())); - assert!(rt::timeout(limit, system::wait_true(arrival)) - .await - .unwrap()); - let old_round = system::lock(&children[1].view.data) - .recovery - .as_ref() - .unwrap() - .meta - .id; - { - let budget = system::lock(&core.budget); - assert!( - budget.attempts.is_empty(), - "an unadmitted generation was charged" - ); - assert_eq!(budget.reserved, 1); - } - children[0] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); - let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) - .await - .unwrap(); - resume.cancel(); - for child in &children { - rt::timeout(limit, child.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); - assert_eq!(child.snapshot().generation, 2); - } - assert!(tree.failure().is_none()); - { - let budget = system::lock(&core.budget); - assert_eq!(budget.attempts.len(), 3); - assert_eq!(budget.reserved, 0); - } - assert!(system.shutdown(limit).await.complete); + async fn supersession_charges_each_selected_round_once() { + crate::test_wait::within( + concat!( + module_path!(), + "::supersession_charges_each_selected_round_once" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system + .async_group("refund-superseded-admission", 2) + .unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::RestForOne, + max_restarts: 3, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for name in ["upstream", "middle", "tail"] { + children.push( + builder + .add(ChildSpec::new(name, &group, |_| async { + Ok::<_, String>(CoordinatorWorker) + })) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + let resume = CancellationToken::new(); + let (entered, arrival) = watch::channel(false); + *system::lock(&core.retry_admission_pause) = Some((resume.clone(), entered)); + children[1] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("middle fails".into())); + assert!(rt::timeout(limit, system::wait_true(arrival)) + .await + .unwrap()); + let old_round = system::lock(&children[1].view.data) + .recovery + .as_ref() + .unwrap() + .meta + .id; + { + let budget = system::lock(&core.budget); + // The round was charged once, at selection; its rebuilds hold + // no reservation of their own. + assert_eq!(budget.attempts.len(), 1); + assert_eq!(budget.reserved, 0); + } + children[0] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("upstream supersedes".into())); + let _new_round = rt::timeout(limit, children[1].view.next_recovery(old_round)) + .await + .unwrap(); + resume.cancel(); + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(child.snapshot().generation, 2); + } + assert!(tree.failure().is_none()); + { + let budget = system::lock(&core.budget); + // Two restart actions: the superseded round and its successor, + // however many children each rebuilt. + assert_eq!(budget.attempts.len(), 2); + assert_eq!(budget.reserved, 0); + } + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] - async fn cohort_budget_retains_v07_per_child_grant_accounting() { - let limit = Duration::from_secs(30); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("cohort-budget-contract", 2).unwrap(); - let mut builder = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 5, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) - .unwrap(); - let mut children = Vec::new(); - for index in 0..6 { - children.push( - builder - .add(ChildSpec::new( - format!("worker-{index}"), - &group, - |_| async { Ok::<_, String>(CoordinatorWorker) }, - )) - .unwrap(), - ); - } - let tree = builder.start().await.unwrap(); - children[5] - .current() - .unwrap() - .child_handle() - .stop_with_reason(ExitReason::Panic("last child fails".into())); - rt::timeout(limit, tree.join()).await.unwrap(); - assert!(tree.failure().unwrap().contains("budget exhausted")); - assert_eq!( - children - .iter() - .map(|child| child.snapshot().generation) - .collect::>(), - vec![2, 2, 2, 2, 2, 1] - ); - assert!(system.shutdown(limit).await.complete); + async fn cohort_round_takes_one_grant_for_all_rebuilt_children() { + crate::test_wait::within( + concat!( + module_path!(), + "::cohort_round_takes_one_grant_for_all_rebuilt_children" + ), + async move { + let limit = Duration::from_secs(30); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("cohort-budget-contract", 2).unwrap(); + let mut builder = system + .supervisor(SupervisorConfig { + strategy: SupervisorStrategy::OneForAll, + max_restarts: 1, + backoff: Backoff { + initial: Duration::ZERO, + maximum: Duration::ZERO, + jitter: false, + ..Backoff::default() + }, + ..SupervisorConfig::default() + }) + .unwrap(); + let mut children = Vec::new(); + for index in 0..6 { + children.push( + builder + .add(ChildSpec::new( + format!("worker-{index}"), + &group, + |_| async { Ok::<_, String>(CoordinatorWorker) }, + )) + .unwrap(), + ); + } + let core = builder.core.clone(); + let tree = builder.start().await.unwrap(); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("last child fails".into())); + // One grant rebuilds all six children (one restart action). + for child in &children { + rt::timeout(limit, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + } + assert!(tree.failure().is_none()); + assert_eq!(system::lock(&core.budget).attempts.len(), 1); + children[5] + .current() + .unwrap() + .child_handle() + .stop_with_reason(ExitReason::Panic("second failure".into())); + rt::timeout(limit, tree.join()).await.unwrap(); + assert!(tree + .failure() + .unwrap() + .contains("budget exhausted by worker-5")); + assert_eq!( + children + .iter() + .map(|child| child.snapshot().generation) + .collect::>(), + vec![2; 6] + ); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test] async fn retry_reservations_exclude_siblings_and_refund_interrupted_backoff() { - let system = ActorSystem::new().unwrap(); - let tree = system - .supervisor(SupervisorConfig { - max_restarts: 1, - ..SupervisorConfig::default() - }) - .unwrap(); - let reservation = tree.core.reserve_retry(1).unwrap(); - assert!(tree.core.reserve_retry(2).is_none()); - assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); - drop(reservation); - let mut replacement = tree.core.reserve_retry(2).unwrap(); - tree.core.close(); - assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); - drop(replacement); - { - let budget = system::lock(&tree.core.budget); - assert!(budget.attempts.is_empty()); - assert_eq!(budget.reserved, 0); - } - drop(tree); - assert!(system.shutdown(Duration::from_secs(10)).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::retry_reservations_exclude_siblings_and_refund_interrupted_backoff" + ), + async move { + let system = ActorSystem::new().unwrap(); + let tree = system + .supervisor(SupervisorConfig { + max_restarts: 1, + ..SupervisorConfig::default() + }) + .unwrap(); + let reservation = tree.core.reserve_retry(1).unwrap(); + assert!(tree.core.reserve_retry(2).is_none()); + assert_eq!(system::lock(&tree.core.budget).attempts.len(), 0); + drop(reservation); + let mut replacement = tree.core.reserve_retry(2).unwrap(); + tree.core.close(); + assert!(!trace::atomic(|| tree.core.retry(&mut replacement))); + drop(replacement); + { + let budget = system::lock(&tree.core.budget); + assert!(budget.attempts.is_empty()); + assert_eq!(budget.reserved, 0); + } + drop(tree); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); + }, + ) + .await; } #[test] diff --git a/concurrency/src/supervision/nested.rs b/concurrency/src/supervision/nested.rs index f35fd1fcb..f78e21188 100644 --- a/concurrency/src/supervision/nested.rs +++ b/concurrency/src/supervision/nested.rs @@ -286,35 +286,44 @@ mod cancellation_tests { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn first_hook_poll_after_admission_closes_is_normal_cancellation() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("closed-nested-hook", 1).unwrap(); - let (entered_tx, entered) = tokio::sync::oneshot::channel(); - let (release, release_rx) = std::sync::mpsc::channel(); - let (scope, lease) = group.reserve(None).unwrap(); - let blocker = scope.handle.spawn(async move { - let _ = entered_tx.send(()); - release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); - }); - entered.await.unwrap(); - let context_owner = ActorRef::spawn_managed( - adapter(system.supervisor(SupervisorConfig::default()).unwrap()), - scope.clone(), - lease, + crate::test_wait::within( + concat!( + module_path!(), + "::first_hook_poll_after_admission_closes_is_normal_cancellation" + ), + async move { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("closed-nested-hook", 1).unwrap(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + let (scope, lease) = group.reserve(None).unwrap(); + let blocker = scope.handle.spawn(async move { + let _ = entered_tx.send(()); + release_rx.recv_timeout(Duration::from_secs(5)).unwrap(); + }); + entered.await.unwrap(); + let context_owner = ActorRef::spawn_managed( + adapter(system.supervisor(SupervisorConfig::default()).unwrap()), + scope.clone(), + lease, + ) + .unwrap(); + let mut hook = adapter(system.supervisor(SupervisorConfig::default()).unwrap()); + let cleanup = hook.builder.as_ref().unwrap().core.clone(); + scope.close_admission(); + // Exercise the narrow gap between the actor's can_start check and the + // first poll of the nested started hook, with real closed scope state. + hook.started(&context_owner.context()).await; + assert!(*cleanup.done.borrow()); + assert!(cleanup.failure.lock().unwrap().is_none()); + scope.stop(ExitReason::Shutdown); + release.send(()).unwrap(); + blocker.await.unwrap(); + context_owner.child_handle().wait_exit_async().await; + assert!(system.shutdown(Duration::from_secs(5)).await.complete); + }, ) - .unwrap(); - let mut hook = adapter(system.supervisor(SupervisorConfig::default()).unwrap()); - let cleanup = hook.builder.as_ref().unwrap().core.clone(); - scope.close_admission(); - // Exercise the narrow gap between the actor's can_start check and the - // first poll of the nested started hook, with real closed scope state. - hook.started(&context_owner.context()).await; - assert!(*cleanup.done.borrow()); - assert!(cleanup.failure.lock().unwrap().is_none()); - scope.stop(ExitReason::Shutdown); - release.send(()).unwrap(); - blocker.await.unwrap(); - context_owner.child_handle().wait_exit_async().await; - assert!(system.shutdown(Duration::from_secs(5)).await.complete); + .await; } } @@ -347,45 +356,63 @@ mod resolve_and_drop_tests { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn resolve_requires_the_current_id_and_an_open_tree() { - let limit = Duration::from_secs(5); - let system = ActorSystem::new().unwrap(); - let tree = running_tree(&system).await; - let (current, other) = (ActorId::next(), ActorId::next()); - let ready: Ready = Arc::new(Mutex::new(Some((current, tree.clone())))); - assert!(resolve(&ready, current).is_ok()); - // An open tree published for another adapter instance never resolves. - assert!(resolve(&ready, other).is_err()); - tree.stop(); - tree.join().await; - // The current adapter's tree, once closed, no longer resolves either. - assert!(resolve(&ready, current).is_err()); - assert!(system.shutdown(limit).await.complete); + crate::test_wait::within( + concat!( + module_path!(), + "::resolve_requires_the_current_id_and_an_open_tree" + ), + async move { + let limit = Duration::from_secs(5); + let system = ActorSystem::new().unwrap(); + let tree = running_tree(&system).await; + let (current, other) = (ActorId::next(), ActorId::next()); + let ready: Ready = Arc::new(Mutex::new(Some((current, tree.clone())))); + assert!(resolve(&ready, current).is_ok()); + // An open tree published for another adapter instance never resolves. + assert!(resolve(&ready, other).is_err()); + tree.stop(); + tree.join().await; + // The current adapter's tree, once closed, no longer resolves either. + assert!(resolve(&ready, current).is_err()); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn dropped_adapter_stops_its_tree_and_clears_only_its_own_entry() { - let limit = Duration::from_secs(5); - let system = ActorSystem::new().unwrap(); - let (mine, newer) = (ActorId::next(), ActorId::next()); + crate::test_wait::within( + concat!( + module_path!(), + "::dropped_adapter_stops_its_tree_and_clears_only_its_own_entry" + ), + async move { + let limit = Duration::from_secs(5); + let system = ActorSystem::new().unwrap(); + let (mine, newer) = (ActorId::next(), ActorId::next()); - // Its own published entry is cleared and its subtree stopped. - let tree = running_tree(&system).await; - let ready: Ready = Arc::new(Mutex::new(Some((mine, tree.clone())))); - drop(adapter(Some(tree.clone()), &ready, Some(mine))); - assert!(system::lock(&ready).is_none()); - assert!(!tree.core.gate.is_open()); + // Its own published entry is cleared and its subtree stopped. + let tree = running_tree(&system).await; + let ready: Ready = Arc::new(Mutex::new(Some((mine, tree.clone())))); + drop(adapter(Some(tree.clone()), &ready, Some(mine))); + assert!(system::lock(&ready).is_none()); + assert!(!tree.core.gate.is_open()); - // A newer generation's entry survives an older adapter's drop. - let replacement = running_tree(&system).await; - let ready: Ready = Arc::new(Mutex::new(Some((newer, replacement.clone())))); - drop(adapter(None, &ready, Some(mine))); - assert_eq!( - system::lock(&ready).as_ref().map(|(id, _)| *id), - Some(newer) - ); - assert!(replacement.core.gate.is_open()); - replacement.stop(); - assert!(system.shutdown(limit).await.complete); + // A newer generation's entry survives an older adapter's drop. + let replacement = running_tree(&system).await; + let ready: Ready = Arc::new(Mutex::new(Some((newer, replacement.clone())))); + drop(adapter(None, &ready, Some(mine))); + assert_eq!( + system::lock(&ready).as_ref().map(|(id, _)| *id), + Some(newer) + ); + assert!(replacement.core.gate.is_open()); + replacement.stop(); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } } @@ -417,56 +444,62 @@ mod stop_panic_tests { /// the adapter's lifecycle helper races it to close the subtree first. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn stopped_hook_stops_the_running_subtree() { - use futures::FutureExt; - let limit = Duration::from_secs(10); - let system = ActorSystem::new().unwrap(); - let group = system.async_group("nested-stop-panic", 1).unwrap(); - let published: Arc>>> = Arc::default(); - let expose = published.clone(); - let placement = group.clone(); - let mut root = system.supervisor(SupervisorConfig::default()).unwrap(); - let subtree = root - .add_supervisor(SupervisorSpec::new( - "subtree", - &group, - SupervisorConfig::default(), - move |tree| { - let leaf = tree.add(ChildSpec::new("leaf", &placement, |_| async { - Ok::<_, String>(Leaf) - }))?; - *system::lock(&expose) = Some(leaf); - Ok::<_, SupervisorError>(()) - }, - )) - .unwrap(); - let root = root.start().await.unwrap(); - let leaf = system::lock(&published).clone().unwrap(); - let mut waiter = Box::pin(leaf.wait_for_generation(leaf.snapshot().generation)); - let waker = std::task::Waker::from(Arc::new(PanicOnWake(AtomicBool::new(false)))); - let mut cx = std::task::Context::from_waker(&waker); - assert!(waiter.as_mut().poll(&mut cx).is_pending()); + crate::test_wait::within( + concat!(module_path!(), "::stopped_hook_stops_the_running_subtree"), + async move { + use futures::FutureExt; + let limit = Duration::from_secs(10); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("nested-stop-panic", 1).unwrap(); + let published: Arc>>> = Arc::default(); + let expose = published.clone(); + let placement = group.clone(); + let mut root = system.supervisor(SupervisorConfig::default()).unwrap(); + let subtree = root + .add_supervisor(SupervisorSpec::new( + "subtree", + &group, + SupervisorConfig::default(), + move |tree| { + let leaf = tree.add(ChildSpec::new("leaf", &placement, |_| async { + Ok::<_, String>(Leaf) + }))?; + *system::lock(&expose) = Some(leaf); + Ok::<_, SupervisorError>(()) + }, + )) + .unwrap(); + let root = root.start().await.unwrap(); + let leaf = system::lock(&published).clone().unwrap(); + let mut waiter = Box::pin(leaf.wait_for_generation(leaf.snapshot().generation)); + let waker = std::task::Waker::from(Arc::new(PanicOnWake(AtomicBool::new(false)))); + let mut cx = std::task::Context::from_waker(&waker); + assert!(waiter.as_mut().poll(&mut cx).is_pending()); - let inner = subtree.current().unwrap(); - let ctx = subtree.child.current().unwrap().context(); - let mut hook = SupervisorActor { - builder: None, - running: Some(inner.clone()), - ready: Arc::default(), - id: None, - shutdown_grace: limit, - parent_core: std::sync::Weak::new(), - }; - let unwound = AssertUnwindSafe(hook.stopped(&ctx)).catch_unwind().await; - let payload = unwound.expect_err("stopped() did not stop the running subtree"); - assert_eq!( - crate::error::panic_message(&*payload), - "waiter waker panicked" - ); - // The backstop stop in Drop is idempotent once the waiter has fired. - drop(hook); - assert!(!inner.core.gate.is_open()); - drop(waiter); - drop(root); - assert!(system.shutdown(limit).await.complete); + let inner = subtree.current().unwrap(); + let ctx = subtree.child.current().unwrap().context(); + let mut hook = SupervisorActor { + builder: None, + running: Some(inner.clone()), + ready: Arc::default(), + id: None, + shutdown_grace: limit, + parent_core: std::sync::Weak::new(), + }; + let unwound = AssertUnwindSafe(hook.stopped(&ctx)).catch_unwind().await; + let payload = unwound.expect_err("stopped() did not stop the running subtree"); + assert_eq!( + crate::error::panic_message(&*payload), + "waiter waker panicked" + ); + // The backstop stop in Drop is idempotent once the waiter has fired. + drop(hook); + assert!(!inner.core.gate.is_open()); + drop(waiter); + drop(root); + assert!(system.shutdown(limit).await.complete); + }, + ) + .await; } } diff --git a/concurrency/src/system.rs b/concurrency/src/system.rs index 0de9b0f51..a2c545c69 100644 --- a/concurrency/src/system.rs +++ b/concurrency/src/system.rs @@ -30,6 +30,7 @@ use group::GroupInner; /// Resource models are opt-in; thread pools reserve one native thread per live /// actor, while pinned shards multiplex actors cooperatively on fixed CPUs. #[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] pub enum ExecutionProfile { AsyncPool { workers: usize, @@ -90,6 +91,7 @@ fn next_admission() -> Result { } #[derive(Debug, Clone, thiserror::Error)] +#[non_exhaustive] pub enum SystemError { #[error("no free persistent actor thread; stop/join a scope before admission")] Capacity, @@ -116,6 +118,7 @@ pub enum SystemError { } #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] pub enum HelperError { #[error("owned helper was cancelled before completion")] Cancelled, @@ -413,8 +416,8 @@ impl ActorSystem { /// Close admission and request ordered shutdown. At expiry the report owns /// pending shutdown state, which can be awaited again. Blocking Rust is not /// forcibly killed and runtime destruction never blocks this caller thread. - pub async fn shutdown(&self, deadline: Duration) -> ShutdownReport { - self.inner.begin_shutdown(deadline); + pub async fn shutdown(&self, grace: Duration) -> ShutdownReport { + self.inner.begin_shutdown(grace); ShutdownReport { pending: Some(self.inner.clone()), complete: false, @@ -425,8 +428,9 @@ impl ActorSystem { supervisor_failures: Vec::new(), group_failures: Vec::new(), coordinator_failures: Vec::new(), + unfinished_jobs: Vec::new(), } - .wait(deadline) + .wait(grace) .await } @@ -523,15 +527,27 @@ impl SystemInner { } } ShutdownTarget::Scope(scope) => { - scope.stop(ExitReason::Shutdown); + // The system is a standalone actor's parent: its + // Shutdown is a parent exit signal (E3). + scope.set_cleanup_timeout(remaining); + scope.signal_parent(ExitReason::Shutdown); let _ = rt::timeout(remaining, scope.joined()).await; } } } - // At grace expiry, no nested child may remain unsignalled behind - // a stuck sibling's longer per-child cleanup timeout. + // The ordered pass ended: every target joined, or the grace + // expired. At grace expiry every remaining actor gets the kill + // (E3), so no nested child remains unsignalled behind a stuck + // sibling's longer per-child cleanup timeout. Before it, the + // remaining scopes only get the parent's Shutdown (a no-op for + // actors already past their death point). + let expired = began.elapsed() >= grace; for scope in scopes.iter().rev() { - scope.stop(ExitReason::Shutdown); + if expired { + scope.kill(); + } else { + scope.signal_parent(ExitReason::Shutdown); + } } // Runtime teardown still requires actual completion, independently // of the ordering grace and the caller's report timeout. @@ -663,8 +679,12 @@ pub struct ShutdownReport { pub group_failures: Vec, /// Coordinator panics recovered through emergency stop-and-join. pub coordinator_failures: Vec, + /// Dead actors whose job cleanup is incomplete: `(actor, running jobs)` + /// still executing after the cleanup deadline (deferred-replies ยง5). + pub unfinished_jobs: Vec<(crate::ActorId, usize)>, } #[derive(Debug, Clone)] +#[non_exhaustive] pub struct ActorExit { pub id: crate::ActorId, pub reason: ExitReason, @@ -683,11 +703,11 @@ pub struct ActorDiagnostic { pub turn_age: Option, } impl ShutdownReport { - pub async fn wait(mut self, deadline: Duration) -> Self { + pub async fn wait(mut self, grace: Duration) -> Self { let Some(owner) = self.pending.as_ref() else { return self; }; - self.complete = rt::timeout(deadline, wait_true(owner.control.done.clone())) + self.complete = rt::timeout(grace, wait_true(owner.control.done.clone())) .await .unwrap_or(false); let state = lock(&owner.state); @@ -705,6 +725,14 @@ impl ShutdownReport { .filter_map(|s| s.diagnostic()) .collect(); self.unfinished_actors = self.unfinished_actor_details.iter().map(|d| d.id).collect(); + self.unfinished_jobs = state + .shutdown_scopes + .iter() + .filter_map(|s| { + let running = s.unfinished_jobs(); + (running > 0).then(|| s.child().map(|c| (c.id(), running)))? + }) + .collect(); self.exits = state .shutdown_scopes .iter() @@ -853,6 +881,11 @@ impl ExecutionGroup { active_turn: Mutex::new(None), ready, done, + cleanup_timeout: std::sync::atomic::AtomicU64::new( + crate::tasks::DEFAULT_STOP_DEADLINE.as_nanos() as u64, + ), + jobs: Mutex::new(None), + bounds: Mutex::new(Default::default()), }); state.scopes.insert(id, scope.clone()); Ok((scope.clone(), Lease { scope, root: true })) @@ -890,8 +923,46 @@ pub(crate) struct Scope { active_turn: Mutex>, ready: watch::Sender, done: watch::Sender, + /// The actor's cleanup deadline (deferred-replies ยง5): the supervisor's + /// `shutdown_timeout` or the system's remaining grace, in nanoseconds. + cleanup_timeout: std::sync::atomic::AtomicU64, + /// The actor's job set, for unfinished-job reporting. + jobs: Mutex>>, + /// Per-`ChildSpec` overrides of `Actor::MAX_OUTSTANDING` and + /// `Actor::MAX_NATIVE_JOBS`, read when the actor instance is created. + bounds: Mutex, } impl Scope { + pub(crate) fn set_bounds(&self, bounds: crate::reply::jobs::Bounds) { + *lock(&self.bounds) = bounds; + } + pub(crate) fn bounds(&self) -> crate::reply::jobs::Bounds { + *lock(&self.bounds) + } + /// Shorten the cleanup deadline that bounds the actor's job join. + pub(crate) fn set_cleanup_timeout(&self, timeout: Duration) { + let nanos = u64::try_from(timeout.as_nanos()).unwrap_or(u64::MAX); + self.cleanup_timeout + .fetch_min(nanos, std::sync::atomic::Ordering::AcqRel); + } + pub(crate) fn cleanup_timeout(&self) -> Duration { + Duration::from_nanos( + self.cleanup_timeout + .load(std::sync::atomic::Ordering::Acquire), + ) + } + pub(crate) fn set_jobs(&self, jobs: std::sync::Weak) { + *lock(&self.jobs) = Some(jobs); + } + /// Jobs still running after the actor's death (job cleanup incomplete). + fn unfinished_jobs(&self) -> usize { + // Upgrade outside the lock: the upgraded set may be the last + // reference, and its destruction must not run under it. + let jobs = lock(&self.jobs).clone(); + jobs.as_ref() + .and_then(std::sync::Weak::upgrade) + .map_or(0, |jobs| jobs.running()) + } pub(crate) fn group(&self) -> ExecutionGroup { ExecutionGroup { inner: self.group.clone(), @@ -932,11 +1003,45 @@ impl Scope { state.ended_at = Some(Instant::now()); } } - link::stop_with_reason(&self.cause, reason, || self.token.cancel()); + link::request_stop(&self.cause, reason, || self.token.cancel()); } pub(crate) fn close_admission(&self) { lock(&self.state).open = false; } + /// The parent's exit signal to this scope's actor (design E3): the + /// supervisor or system is the parent. Admission closes for the managed + /// tree at once (B), but nothing is committed for the actor: a + /// non-trapping actor begins exit with `reason` without `stopped()` + /// (`Normal` is ignored), a trapping one gets a parent exit queued behind + /// its messages, which ends it through `stopped()`. Before the actor is + /// attached this is an ordinary stop. + pub(crate) fn signal_parent(&self, reason: ExitReason) { + let child = { + let mut state = lock(&self.state); + state.open = false; + if state.ended_at.is_none() { + state.ended_at = Some(Instant::now()); + } + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, reason, true), + None => self.stop(reason), + } + } + /// The parent's kill (E3 escalation at `shutdown_timeout` or grace): the + /// actor ends `Killed`, skipping a `stopped()` that has not begun. + pub(crate) fn kill(&self) { + let child = { + let mut state = lock(&self.state); + state.open = false; + state.actor.clone() + }; + match child { + Some(child) => child.exit_from(crate::ActorId::EXTERNAL, ExitReason::Kill, true), + None => self.stop(ExitReason::Killed), + } + } pub(crate) fn register( &self, name: &str, @@ -1074,6 +1179,11 @@ impl Scope { pub(crate) fn actor_lease(self: &Arc) -> Result { self.lease(true) } + /// A blocking job's lease, held until its native body ends: the scope + /// is not joined (cleanup incomplete) while the work still runs. + pub(crate) fn job_lease(self: &Arc) -> Result { + self.lease(false) + } /// Dispatch the actor root to its reserved worker or runtime shard. pub(crate) fn dispatch_actor(&self, future: F) -> Result<(), SystemError> where @@ -1227,8 +1337,12 @@ impl Drop for Lease { .scope .child() .and_then(|c| c.exit_reason()) - .or_else(|| lock(&self.scope.cause).clone()) - .unwrap_or(ExitReason::Kill); + .or_else(|| { + crate::link::lock_request(&self.scope.cause) + .reason() + .cloned() + }) + .unwrap_or(ExitReason::Killed); self.scope.ready.send_replace(StartupState::Failed(reason)); } } @@ -1301,70 +1415,88 @@ mod coordinator_tests { } #[tokio::test] async fn coordinator_fault_stops_all_targets_and_retains_owned_cleanup() { - use crate::tasks::ActorStart; - let system = ActorSystem::new().unwrap(); - let group = system.async_group("coordinator-fault", 1).unwrap(); - let stopped = Arc::new(AtomicUsize::new(0)); - let first = Worker(stopped.clone()).start_on(&group).unwrap(); - let second = Worker(stopped.clone()).start_on(&group).unwrap(); - first.ready().await.unwrap(); - second.ready().await.unwrap(); - let (done, _) = watch::channel(false); - let fault = Arc::new(FaultUnit { - allow_emergency: AtomicBool::new(false), - done, - }); - system.inner.register_unit(fault.clone()).unwrap(); - let mut report = system.shutdown(Duration::from_millis(30)).await; - assert!(!report.complete); - // The report is a snapshot, not an acknowledgement that the coordinator - // ran. Under full-suite load its 30ms deadline can precede the fault. - // Refresh with bounded, yielding waits until that diagnostic is visible. - report = rt::timeout(LIMIT, async move { - while !report - .coordinator_failures - .iter() - .any(|e| e.contains("injected system coordinator fault")) - { - // Yield even if a regressed coordinator closes its done watch: - // wait() can then return immediately, and LIMIT must still fire. - rt::sleep(Duration::from_millis(1)).await; - report = report.wait(Duration::ZERO).await; - } - report - }) - .await - .expect("coordinator did not report its injected failure"); - assert!(report - .coordinator_failures - .iter() - .any(|e| e.contains("injected system coordinator fault"))); - tokio::time::timeout(LIMIT, first.join()).await.unwrap(); - tokio::time::timeout(LIMIT, second.join()).await.unwrap(); - assert_eq!(stopped.load(Ordering::SeqCst), 2); - assert!(!*system.inner.control.done.borrow()); - drop(system); // pending report + coordinator retain runtime ownership - fault.allow_emergency.store(true, Ordering::SeqCst); - let report = report.wait(LIMIT).await; - assert!(report.complete); - assert!(!report.coordinator_failures.is_empty()); - assert!(report.coordinator_failures.len() <= 16); + crate::test_wait::within( + concat!( + module_path!(), + "::coordinator_fault_stops_all_targets_and_retains_owned_cleanup" + ), + async move { + use crate::tasks::ActorStart; + let system = ActorSystem::new().unwrap(); + let group = system.async_group("coordinator-fault", 1).unwrap(); + let stopped = Arc::new(AtomicUsize::new(0)); + let first = Worker(stopped.clone()).start_on(&group).unwrap(); + let second = Worker(stopped.clone()).start_on(&group).unwrap(); + first.ready().await.unwrap(); + second.ready().await.unwrap(); + let (done, _) = watch::channel(false); + let fault = Arc::new(FaultUnit { + allow_emergency: AtomicBool::new(false), + done, + }); + system.inner.register_unit(fault.clone()).unwrap(); + let mut report = system.shutdown(Duration::from_millis(30)).await; + assert!(!report.complete); + // The report is a snapshot, not an acknowledgement that the coordinator + // ran. Under full-suite load its 30ms deadline can precede the fault. + // Refresh with bounded, yielding waits until that diagnostic is visible. + report = rt::timeout(LIMIT, async move { + while !report + .coordinator_failures + .iter() + .any(|e| e.contains("injected system coordinator fault")) + { + // Yield even if a regressed coordinator closes its done watch: + // wait() can then return immediately, and LIMIT must still fire. + rt::sleep(Duration::from_millis(1)).await; + report = report.wait(Duration::ZERO).await; + } + report + }) + .await + .expect("coordinator did not report its injected failure"); + assert!(report + .coordinator_failures + .iter() + .any(|e| e.contains("injected system coordinator fault"))); + tokio::time::timeout(LIMIT, first.join()).await.unwrap(); + tokio::time::timeout(LIMIT, second.join()).await.unwrap(); + assert_eq!(stopped.load(Ordering::SeqCst), 2); + assert!(!*system.inner.control.done.borrow()); + drop(system); // pending report + coordinator retain runtime ownership + fault.allow_emergency.store(true, Ordering::SeqCst); + let report = report.wait(LIMIT).await; + assert!(report.complete); + assert!(!report.coordinator_failures.is_empty()); + assert!(report.coordinator_failures.len() <= 16); + }, + ) + .await; } #[tokio::test] async fn emergency_cleanup_does_not_resignal_an_already_finished_faulting_unit() { - let system = ActorSystem::new().unwrap(); - let (done, _) = watch::channel(false); - let fault = Arc::new(FaultUnit { - allow_emergency: AtomicBool::new(false), - done, - }); - system.inner.register_unit(fault.clone()).unwrap(); - let report = system.shutdown(Duration::from_millis(30)).await; - assert!(!report.complete); - // Another owner finishes it; its stop method remains permanently broken. - fault.done.send_replace(true); - let report = report.wait(LIMIT).await; - assert!(report.complete); - assert!(!report.coordinator_failures.is_empty()); + crate::test_wait::within( + concat!( + module_path!(), + "::emergency_cleanup_does_not_resignal_an_already_finished_faulting_unit" + ), + async move { + let system = ActorSystem::new().unwrap(); + let (done, _) = watch::channel(false); + let fault = Arc::new(FaultUnit { + allow_emergency: AtomicBool::new(false), + done, + }); + system.inner.register_unit(fault.clone()).unwrap(); + let report = system.shutdown(Duration::from_millis(30)).await; + assert!(!report.complete); + // Another owner finishes it; its stop method remains permanently broken. + fault.done.send_replace(true); + let report = report.wait(LIMIT).await; + assert!(report.complete); + assert!(!report.coordinator_failures.is_empty()); + }, + ) + .await; } } diff --git a/concurrency/src/tasks/actor.rs b/concurrency/src/tasks/actor.rs index d8aed1c50..4c21ebc97 100644 --- a/concurrency/src/tasks/actor.rs +++ b/concurrency/src/tasks/actor.rs @@ -1,31 +1,39 @@ +use crate::child_handle::Completion; use crate::child_handle::{ActorId, ChildHandle}; +use crate::error::NoProc; use crate::error::{panic_message, ActorError, ExitReason}; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkTable, - LinkedExitReason, SendExitFn, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, LinkTable, + LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::mailbox::tasks as mpsc; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; +use crate::reply::{registry::Registry, reply_slot, Caller, Slot, SlotHandle}; +use crate::reply::{Deadline, PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use core::pin::pin; use futures::future::{self, FutureExt as _}; use spawned_rt::{ - tasks::{self as rt, oneshot, timeout, watch, CancellationToken, JoinHandle}, + tasks::{self as rt, watch, CancellationToken, JoinHandle}, threads, }; use std::sync::atomic::Ordering; +#[cfg(test)] +use std::{collections::HashMap, sync::Mutex}; use std::{ - collections::HashMap, - fmt::Debug, - future::Future, - panic::AssertUnwindSafe, - pin::Pin, - sync::{Arc, Mutex}, - time::Duration, + fmt::Debug, future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Duration, }; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; +/// Cleanup deadline for actors without a supervisor or system grace: the +/// bound on joining an exiting actor's cancelled jobs. +pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); + +#[path = "jobs.rs"] +mod jobs; +pub use jobs::{BlockingJobPermit, JobCtx, JobPermit}; + // --------------------------------------------------------------------------- // Backend // --------------------------------------------------------------------------- @@ -43,6 +51,7 @@ pub use crate::response::DEFAULT_REQUEST_TIMEOUT; /// account for those threads and reactor descriptors when sizing populations. /// Actor completion does not wait for detached helpers or runtime destruction. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[non_exhaustive] pub enum Backend { #[default] Async, @@ -61,6 +70,21 @@ pub enum Backend { /// /// Actors must be `Send + Sized + 'static` so they can be moved to a spawned task. pub trait Actor: Send + Sized + 'static { + /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running + /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. + /// Default 1024; capped at 65535. + const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + + /// Native threads this instance's blocking jobs may run at once + /// (`pipe_blocking*`). `None`: `min(4, available_parallelism)`. + /// + /// The default is small: an actor that runs many blocking jobs at once + /// (I/O, proofs, signature checks) usually needs more, and under load the + /// only symptom is `PipeError::NativeBudget` refusals (the first one + /// per instance is logged as a `native_budget_exhausted` warning). Set it + /// with `#[actor(max_native_jobs = N)]` or `ChildSpec::max_native_jobs`. + const MAX_NATIVE_JOBS: Option = None; + fn started(&mut self, _ctx: &Context) -> impl Future + Send { async {} } @@ -86,10 +110,32 @@ pub trait Actor: Send + Sized + 'static { /// Per-message handler trait. Implement once for each message type the actor handles. /// +/// `handle` is the envelope dispatch for both `send` and `request`. Answer +/// through `reply`: [`PendingReply::reply`] now, or [`PendingReply::defer`] to +/// answer from a later turn, a job or `stopped()`. A request whose +/// `PendingReply` is dropped unanswered gets `ReplyDropped`. A sent message +/// has a detached `PendingReply` (no caller). +/// +/// `#[actor]` generates this from `#[request_handler]`, `#[send_handler]` and +/// `#[deferred_handler]` methods. Manual impls: +/// +/// ```ignore +/// impl Handler for Counter { +/// async fn handle(&mut self, _: GetCount, reply: PendingReply<'_, u64>, _: &Context) { +/// reply.reply(self.count); +/// } +/// } +/// ``` +/// /// Uses RPITIT (return-position `impl Trait` in traits), which means this trait /// is **not** object-safe. For type-erased references, use [`Receiver`] / [`Recipient`]. pub trait Handler: Actor { - fn handle(&mut self, msg: M, ctx: &Context) -> impl Future + Send; + fn handle( + &mut self, + msg: M, + reply: PendingReply<'_, M::Result>, + ctx: &Context, + ) -> impl Future + Send; } // --------------------------------------------------------------------------- @@ -97,6 +143,11 @@ pub trait Handler: Actor { // --------------------------------------------------------------------------- trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } #[cfg(any(feature = "diagnostics", spawned_sim))] fn message_type(&self) -> &'static str { std::any::type_name::() @@ -108,18 +159,91 @@ trait Envelope: Send { #[cfg(feature = "metrics")] fn reject(self: Box) {} - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>>; + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self, _replies: Option<&Registry>) {} + + /// Dispatch once: the turn's future, and the request's caller if any. + /// The envelope keeps the request's slot for the whole turn and lends + /// it to the handler's `PendingReply` (no reference count change per + /// message); the run loop answers the lent caller at the death point if + /// the runtime destroys the turn before it finishes. + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a>; +} + +/// A dispatched turn: its future and the caller it answers, lent by the +/// envelope for the turn. +type Dispatch<'a> = ( + Pin + Send + 'a>>, + Option<&'a dyn Caller>, +); + +/// Dispatch `envelope`, catching a panic while the handler builds its +/// future (a manual `Handler` may panic before it returns one): a handler +/// panic like any other. +fn dispatch<'a, A: Actor>( + envelope: &'a mut (dyn Envelope + Send), + actor: &'a mut A, + ctx: &'a Context, +) -> Result, Box> { + let mut parts = Some((envelope, actor)); + std::panic::catch_unwind(AssertUnwindSafe(move || { + let (envelope, actor) = parts.take().expect("dispatched once"); + envelope.handle(actor, ctx) + })) +} + +/// Take an envelope's payload at its one dispatch. +fn dispatched(payload: &mut Option) -> T { + payload.take().expect("an envelope is dispatched once") +} + +/// Answer every request queued in a dead owner's mailbox, destroying +/// nothing: the drained envelopes come back, to be destroyed once the death +/// point's wake-ups ran. Admission is closed first, so nothing is queued +/// after. +fn answer_mailbox( + rx: &mut mpsc::Receiver + Send>>, + replies: Option<&Registry>, +) -> Vec + Send>> { + let mut queued = rx.drain(); + for envelope in &mut queued { + envelope.discharge(replies); + } + queued +} + +/// A never-polled actor's startup payload, dropped with its runtime. +type StartupPayload = ( + A, + Context, + mpsc::Receiver + Send>>, + CancellationToken, +); +fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + let replies = payload.1.shared.replies.clone(); + erase(answer_mailbox(&mut payload.2, Some(&replies))) +} +fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() } struct MessageEnvelope { - msg: M, - tx: Option>, + // Before the message: a plain drop also answers the request first. + slot: SlotHandle, + /// The slot lent to the dispatched turn (taken from `slot`). + lent: Option>>, + msg: Option, #[cfg(feature = "metrics")] observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + from: u64, } impl Envelope for MessageEnvelope @@ -127,6 +251,15 @@ where A: Actor + Handler, M: Message, { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } #[cfg(any(feature = "diagnostics", spawned_sim))] fn message_type(&self) -> &'static str { std::any::type_name::() @@ -141,73 +274,264 @@ where observation.reject(); } } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { - let result = actor.handle(self.msg, ctx).await; - if let Some(tx) = self.tx { - let delivered = tx.send(result).is_ok(); - #[cfg(feature = "metrics")] - crate::instrumentation::reply(delivered); - #[cfg(not(feature = "metrics"))] - let _ = delivered; + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { + slot, lent, msg, .. + } = self; + let msg = dispatched(msg); + *lent = slot.take(); + let lent: &'a Option>> = lent; + let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); + ( + Box::pin(actor.handle(msg, reply, ctx)), + lent.as_ref().map(|slot| slot as &dyn Caller), + ) + } +} + +/// A sent message (no caller): no reply slot to hold, lend or dispose of, +/// so its dispatch builds only a detached `PendingReply`. +struct SendEnvelope { + msg: Option, + #[cfg(feature = "metrics")] + observation: Option, + #[cfg(feature = "lifecycle-trace")] + from: u64, +} + +impl Envelope for SendEnvelope +where + A: Actor + Handler, + M: Message, +{ + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + #[inline] + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let msg = dispatched(&mut self.msg); + ( + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)), + None, + ) + } +} + +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(any(feature = "diagnostics", spawned_sim))] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let down = dispatched(&mut self.down); + let turn = Box::pin(async move { + fail_point!("monitor.down_before_dispatch", async); + if crate::monitor::fence::take_queued(&ctx.shared.monitors, &down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.shared.replies); + actor.handle(down, reply, ctx).await; } - }) + }); + (turn, None) } } +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. fn enqueue( sender: &mpsc::Sender + Send>>, _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, msg: M, - tx: Option>, + slot: SlotHandle, ) -> Result<(), ActorError> where A: Actor + Handler, M: Message, { + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); let envelope: Box + Send> = Box::new(MessageEnvelope { - msg, - tx, + msg: Some(msg), + slot, + lent: None, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// Admit a sent message (no caller): no reply slot at all. +fn enqueue_send( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + msg: M, +) -> Result<(), ActorError> +where + A: Actor + Handler, + M: Message, +{ + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), true); + let envelope: Box + Send> = Box::new(SendEnvelope { + msg: Some(msg), + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +struct DownEnvelope { + down: Option, + #[cfg(feature = "metrics")] + observation: Option, +} + +fn enqueue_down>( + sender: &mpsc::Sender + Send>>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down: Some(down), #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - match sender.send(envelope) { - Ok(()) => { + admit::(sender, replies, envelope, 0) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +fn admit( + sender: &mpsc::Sender + Send>>, + replies: &Registry, + envelope: Box + Send>, + _from: u64, +) -> Result<(), ActorError> { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - Ok(()) + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false, _from); + return Err(ActorError::NoProc); } - Err(_error) => { - #[cfg(feature = "metrics")] - { - _error.0.reject(); + match sender.send(envelope) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true, _from); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + _error.0.reject(); + } + crate::lifecycle_trace::life_send::(owner, false, _from); + Err(ActorError::NoProc) } - Err(ActorError::ActorStopped) } + }) +} + +/// A closure run in one of the actor's turns (`spawned_test::tasks::in_actor`). +type InTurn = Box) -> R + Send>; +struct InTurnEnvelope { + slot: SlotHandle, + lent: Option>>, + f: Option>, +} + +impl Envelope for InTurnEnvelope { + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "InTurn" + } + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { slot, lent, f } = self; + let f = dispatched(f); + *lent = slot.take(); + let lent: &'a Option>> = lent; + // Inside the future, so a panicking closure is a handler panic. + let turn = Box::pin(async move { + let reply = PendingReply::lent(lent.as_ref(), &ctx.shared.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); + }); + (turn, lent.as_ref().map(|slot| slot as &dyn Caller)) } } // Private lifecycle signal, excluded from application message/handler metrics. // Nested supervisor adapters have no public mailbox traffic to bury this signal. -struct FailureEnvelope(String); +struct FailureEnvelope(Option); impl Envelope for FailureEnvelope { #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "Exit" } - fn handle<'a>( - self: Box, - _: &'a mut A, - _: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { - panic!("nested supervisor failed: {}", self.0); - }) + fn handle<'a>(&'a mut self, _: &'a mut A, _: &'a Context) -> Dispatch<'a> { + let failure = dispatched(&mut self.0); + ( + Box::pin(async move { + panic!("nested supervisor failed: {failure}"); + }), + None, + ) } } @@ -216,26 +540,130 @@ impl Envelope for FailureEnvelope { /// `Handler` on the actor โ€” every actor can receive `Exit` via the /// trait's default-no-op `exit_received` callback. struct ExitEnvelope { - exit: Exit, + exit: Option, } impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "Exit" + } + + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + ( + Box::pin(async move { + actor.exit_received(exit, ctx).await; + }), + None, + ) + } +} + +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +struct ParentExitEnvelope { + exit: Option, +} + +impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + "ParentExit" + } + + fn handle<'a>(&'a mut self, _actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + (Box::pin(async move { ctx.parent_exit(exit.reason) }), None) + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +struct HeldEnvelope { + exit: Option, + parent: bool, +} + +impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } #[cfg(feature = "diagnostics")] fn message_type(&self) -> &'static str { "Exit" } - fn handle<'a>( - self: Box, - actor: &'a mut A, - ctx: &'a Context, - ) -> Pin + Send + 'a>> { - Box::pin(async move { - actor.exit_received(self.exit, ctx).await; - }) + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let exit = dispatched(&mut self.exit); + let parent = self.parent; + let turn = Box::pin(async move { + if !link::lock_request(&ctx.shared.linked_reason).held_trapped() { + return; + } + if parent { + ctx.parent_exit(exit.reason); + } else { + actor.exit_received(exit, ctx).await; + } + }); + (turn, None) } } +/// Build the signal target of a new actor instance. +fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender + Send>>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: CancellationToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit: Some(exit) }); + exit_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = + Box::new(ParentExitEnvelope { exit: Some(exit) }); + parent_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { + exit: Some(exit), + parent, + }); + held_tx.send(envelope).map_err(|_| ActorError::NoProc) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} + // --------------------------------------------------------------------------- // Context // --------------------------------------------------------------------------- @@ -264,8 +692,17 @@ struct ActorShared { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + /// This instance as the target of exit signals and link notices. + signal: Arc, mailbox: crate::instrumentation::Mailbox, managed: Option>, + /// This instance's reply registry: deferred tokens, credits, turn phase. + replies: Arc, + /// This instance's pipe-to-self jobs. + jobs: Arc, + /// Becomes `true` when cleanup completes (death plus the bounded job + /// join); closed if the actor task is dropped. `join()` waits for it. + cleaned: watch::Receiver, } impl Clone for Context { @@ -283,9 +720,8 @@ impl Debug for Context { } impl Context { - /// Create a `Context` from an `ActorRef`. Useful for setting up timers - /// or stream listeners from outside the actor. - pub fn from_ref(actor_ref: &ActorRef) -> Self { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { Self { shared: actor_ref.shared.clone(), } @@ -299,46 +735,119 @@ impl Context { self.shared.mailbox.depth() } + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. + pub fn deferred_outstanding(&self) -> usize { + self.shared.replies.outstanding() + } + /// The actor's unique identity. pub fn id(&self) -> ActorId { self.shared.id } - /// Signal the actor to stop. The current handler will finish, then - /// `stopped()` is called and the actor exits. + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. pub fn stop(&self) { self.stop_with_reason(ExitReason::Normal); } - /// Request a cooperative exit with an explicit reason, without unwinding. - /// The current handler finishes and `stopped()` runs. The first committed - /// stop/link reason wins; a cleanup panic can still make the exit abnormal. - /// `Shutdown` is non-abnormal; `Kill` requests an abnormal exit and can - /// trigger transient supervision or propagate through links. It does not - /// forcibly interrupt running Rust code. + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(&self.shared.linked_reason, reason, || { + link::stop_with_reason(self.shared.id, &self.shared.linked_reason, reason, || { + self.shared.cancellation_token.cancel() + }); + } + + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.shared.linked_reason, reason, || { self.shared.cancellation_token.cancel() }); } + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. A signal to this actor's own handle is a signal, not a + /// self-exit: a non-trapping actor ignores `Normal`. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.shared.replies.signal_permit() else { + return; + }; + if link::lock_request(&self.shared.linked_reason).is_dead() { + return; + } + fail_point!("actor.exit_before_deliver"); + target.exit_from(self.shared.id, reason, false); + } + /// Send a fire-and-forget message to this actor. pub fn send(&self, msg: M) -> Result<(), ActorError> where A: Handler, M: Message, { - enqueue(&self.shared.sender, &self.shared.mailbox, msg, None) + enqueue_send( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; Ok(rx) } @@ -363,14 +872,18 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { - Err(error) => Err(error), - Ok(rx) => match timeout(duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let rx = match self.send_request_with(msg, crate::reply::clock::after(duration), false) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.into_timed_future_after(duration).await; observation.finish(&result); result } @@ -395,10 +908,12 @@ impl Context { /// Set up a unidirectional monitor on another actor. /// /// Returns a [`MonitorRef`] that can be used to cancel the monitor via - /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] - /// message is delivered to this actor's mailbox via `Handler`. + /// [`Context::demonitor`]. When the monitored actor's exit is published, + /// a [`Down`] with its final reason is delivered via `Handler`. /// - /// If the target is already dead, a `Down` message is delivered immediately. + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`; use + /// [`ChildHandle::exit_reason`] for the real reason). /// /// Multiple independent monitors are allowed on the same target โ€” each /// call returns a distinct `MonitorRef`. @@ -415,22 +930,53 @@ impl Context { let monitor_ref = MonitorRef::next(); let active = spawned_rt::tasks::CancellationToken::new(); - { - let mut monitors = self - .shared - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()); - // The token check makes registration inert once shutdown starts. - // The closed table below serializes registration with exit cleanup, - // even on paths that unwind before cancelling the owner token. - if self.shared.cancellation_token.is_cancelled() { - return monitor_ref; - } - let Some(monitors) = monitors.as_mut() else { - return monitor_ref; + let observer = self.shared.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) }; - monitors.insert(monitor_ref, active.clone()); + crate::monitor::fence::register( + &self.shared.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.shared.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = enqueue_down( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + Down { + monitor_ref, + actor: target.id(), + reason: ExitReason::NoProc, + }, + ); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active + }); + if !registered { + return monitor_ref; } let target = target.clone(); @@ -449,21 +995,31 @@ impl Context { futures::future::Either::Left(_) => return, futures::future::Either::Right((reason, _)) => reason, }; - // Remove the entry from the monitor table so it doesn't accumulate - // stale entries over the actor's lifetime. Done before delivery - // since `demonitor` is now a no-op for this monitor anyway. - let deliver = monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some(); - if deliver { - let _ = actor_ref.send(Down { - monitor_ref, - reason, + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = enqueue_down( + &actor_ref.shared.sender, + &actor_ref.shared.mailbox, + &actor_ref.shared.replies, + Down { + monitor_ref, + actor: target.id(), + reason, + }, + ); }); - } + }); }); monitor_ref @@ -471,62 +1027,71 @@ impl Context { /// Cancel a previously-set monitor. /// - /// The monitor waiter is released promptly. - /// If the target hasn't yet died, no `Down` message will be delivered. - /// If a `Down` message has already been delivered (or queued), this is - /// a best-effort cancellation โ€” the message may still arrive. + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. pub fn demonitor(&self, monitor_ref: MonitorRef) { - let active = self - .shared - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); + let observer = self.shared.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::demonitor(&self.shared.monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + }) + }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { active.cancel(); } } - /// Set up a bidirectional link with another actor. + /// Set up a bidirectional link with another actor (OTP `link/1`). /// - /// When either side dies abnormally, the other receives an exit signal. - /// By default the receiver is terminated by the signal. Call - /// [`trap_exit(true)`] to convert signals into `Exit` messages delivered - /// via [`Actor::exit_received`] instead. + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. /// - /// If the target is already dead with an abnormal reason, the exit - /// signal is delivered immediately without retaining a link. Repeating an - /// existing live link is a no-op; each attempt to link a closed peer reports - /// that peer's exit for the new attempt. + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc` (decision 1). Use [`try_link`](Self::try_link) to get + /// `Err(NoProc)` instead. An existing link, or a pending notice from that + /// peer, makes this a no-op. Inert once this actor's exit began (in + /// `stopped()`, from a helper after the stop, or past its death point): + /// no record and no notice, as `monitor`. /// - /// [`trap_exit(true)`]: Self::trap_exit /// [`Actor::exit_received`]: Actor::exit_received pub fn link(&self, target: &ChildHandle) { - let own_signal = link::make_signal( - self.shared.trap_exit.clone(), - self.own_cancel_fn(), - self.own_send_exit_fn(), - self.shared.linked_reason.clone(), - ); - let peer_signal = link::make_signal( - target.trap_exit_flag().clone(), - target.cancel_fn().clone(), - target.send_exit_fn().clone(), - target.linked_reason().clone(), - ); link::register_link( - self.shared.id, + &self.shared.signal, &self.shared.links, - own_signal, - target.id(), + target.signal_target(), target.links(), - peer_signal, + false, ); } - /// Remove a previously-set bidirectional link. + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record, whether or not this actor + /// traps exits. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.shared.signal, + &self.shared.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor, including a notice already taken at + /// the peer's death; an `Exit` already queued stays. pub fn unlink(&self, target: &ChildHandle) { link::unregister_link( self.shared.id, @@ -536,31 +1101,55 @@ impl Context { ); } - /// Control how exit signals from linked actors are handled. + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. /// - /// - `false` (default): an abnormal exit signal from a linked actor cancels - /// this actor (propagating the death). - /// - `true`: the signal is converted to an `Exit` message and delivered to - /// [`Actor::exit_received`]. + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. /// - /// A direct `stop_with_reason(Kill)` cannot be trapped. Propagated link - /// notices, including Kill, are delivered as Exit when trapping is enabled. - /// Cancellation remains cooperative: an in-flight handler is not preempted. - pub fn trap_exit(&self, enabled: bool) { - self.shared.trap_exit.store(enabled, Ordering::Release); - } - - /// Build a type-erased cancel closure for this actor. - fn own_cancel_fn(&self) -> Arc { - let token = self.shared.cancellation_token.clone(); - let reason = self.shared.linked_reason.clone(); - Arc::new(move || link::normal_stop(&reason, || token.cancel())) + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.shared.linked_reason).exit_pending() } - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn own_send_exit_fn(&self) -> SendExitFn { - self.shared.send_exit_fn() + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). + /// + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for + /// [`Actor::exit_received`]. + /// + /// Only a kill is untrappable. A parent's exit (the `start_linked` parent, + /// the supervisor or the system) ends a trapping actor normally, through + /// `stopped()`, and never reaches `exit_received`. Set it first thing in + /// `started()`: signals that arrive while `started()` runs are resolved + /// when it returns. Handlers are never preempted. + pub fn trap_exit(&self, enabled: bool) { + let id = self.shared.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.shared.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); } /// Type-erased handle to this actor, without an intermediate `ActorRef`. @@ -643,6 +1232,9 @@ impl Context { } } +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + // Bridge: Context implements Receiver for any M that A handles impl Receiver for Context where @@ -653,8 +1245,20 @@ where Context::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - Context::request_raw(self, msg) + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline.ticks(), false) } } @@ -665,10 +1269,28 @@ where /// Object-safe trait for sending a single message type to an actor. /// /// Implemented automatically by `ActorRef` and `Context` for any -/// message type that `A` handles. -pub trait Receiver: Send + Sync { +/// message type that `A` handles. Sealed: it cannot be implemented outside +/// spawned (a method can then be added without a breaking change). +pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; - fn request_raw(&self, msg: M) -> Result, ActorError>; + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its + /// deadline is fixed now (the slot carries it). On a [`Recipient`], + /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError>; + /// `send_request` for the `request` helper, which arms its own timer: + /// the deadline does not go to the owner's heap. + #[doc(hidden)] + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + self.send_request(msg, deadline) + } } /// Type-erased reference for sending a single message type. @@ -683,14 +1305,17 @@ pub async fn request( timeout_duration: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.request_raw(msg) { - Err(error) => Err(error), - Ok(rx) => match timeout(timeout_duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + // The deadline is computed before enqueueing (send time, as OTP's call). + // Bind the receiver before awaiting it (see `ActorRef::request_with_timeout`). + let rx = match recipient.send_request_timed(msg, Deadline::after(timeout_duration)) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.into_timed_future_after(timeout_duration).await; observation.finish(&result); result } @@ -741,7 +1366,10 @@ impl ActorRef { .await } pub(crate) fn signal_failure(&self, error: String) { - let _ = self.shared.sender.send(Box::new(FailureEnvelope(error))); + let _ = self + .shared + .sender + .send(Box::new(FailureEnvelope(Some(error)))); } /// Send a fire-and-forget message to the actor. pub fn send(&self, msg: M) -> Result<(), ActorError> @@ -749,17 +1377,53 @@ impl ActorRef { A: Handler, M: Message, { - enqueue(&self.shared.sender, &self.shared.mailbox, msg, None) + enqueue_send( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.shared.sender, &self.shared.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline, Some(&self.shared.replies), raw); + enqueue( + &self.shared.sender, + &self.shared.mailbox, + &self.shared.replies, + msg, + slot, + )?; Ok(rx) } @@ -773,6 +1437,31 @@ impl ActorRef { .await } + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: `spawned_test::tasks::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.shared.replies), false); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Some(Box::new(f)), + slot, + lent: None, + }); + // Past the death point the actor is gone (E4), as for `enqueue`. + if self.shared.replies.refuses_admission() { + return Err(ActorError::NoProc); + } + self.shared + .sender + .send(envelope) + .map_err(|_| ActorError::NoProc)?; + Ok(rx) + } + /// Send a request and wait for the reply with a custom timeout. pub async fn request_with_timeout( &self, @@ -784,14 +1473,18 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { - Err(error) => Err(error), - Ok(rx) => match timeout(duration, rx).await { - Ok(Ok(result)) => Ok(result), - Ok(Err(_)) => Err(ActorError::ActorStopped), - Err(_) => Err(ActorError::RequestTimeout), - }, + // Bind the receiver before awaiting it: awaiting inside a `match` on + // the `Result` keeps the scrutinee's storage alive beside the awaited + // receiver and doubles this future's size. + let rx = match self.send_request_with(msg, crate::reply::clock::after(duration), false) { + Ok(rx) => rx, + Err(error) => { + let result = Err(error); + observation.finish(&result); + return result; + } }; + let result = rx.into_timed_future_after(duration).await; observation.finish(&result); result } @@ -805,34 +1498,43 @@ impl ActorRef { Arc::new(self.clone()) } - /// Get a `Context` from this ref, for timer setup or stream listeners. + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. pub fn context(&self) -> Context { Context::from_ref(self) } /// Wait until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline; jobs still running then + /// are reported, not awaited). A `stopped()` or state destructor that + /// never returns keeps it waiting; bound it with a timeout, or use + /// [`wait_exit`](Self::wait_exit) to observe death alone. pub async fn join(&self) { let _ = self.wait_exit().await; + let mut cleaned = self.shared.cleaned.clone(); + loop { + if *cleaned.borrow_and_update() { + return; + } + if cleaned.changed().await.is_err() { + return; + } + } } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods (for example, after + /// runtime shutdown). pub fn exit_reason(&self) -> Option { crate::child_handle::poll_exit_reason(&self.shared.completion_rx) } - /// Wait until the actor stops and return the exit reason. + /// Wait until the actor's exit is published and return its final reason. pub async fn wait_exit(&self) -> ExitReason { - let mut rx = self.shared.completion_rx.clone(); - loop { - if let Some(reason) = rx.borrow_and_update().clone() { - return reason; - } - if rx.changed().await.is_err() { - return ExitReason::Kill; - } - } + crate::child_handle::wait_loop(self.shared.completion_rx.clone()).await } /// Waiting messages, including in-flight enqueue reservations, excluding @@ -861,25 +1563,11 @@ impl ActorRef { } impl ActorShared { - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn send_exit_fn(&self) -> SendExitFn { - let sender = self.sender.clone(); - Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender.send(envelope).map_err(|_| ActorError::ActorStopped) - }) - } - fn child_handle(&self) -> ChildHandle { - ChildHandle::from_tasks( - self.id, - self.cancellation_token.clone(), - self.completion_rx.clone(), - self.trap_exit.clone(), + ChildHandle::new( + self.signal.clone(), + Completion::Tasks(self.completion_rx.clone()), self.links.clone(), - self.linked_reason.clone(), - self.send_exit_fn(), ) .with_mailbox_failures(self.sender.failures()) } @@ -901,8 +1589,20 @@ where ActorRef::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - ActorRef::request_raw(self, msg) + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline.ticks(), false) } } @@ -940,7 +1640,7 @@ impl ActorRef { .map(|s| s.token.clone()) .unwrap_or_default(); let (completion_tx, completion_rx) = watch::channel(None); - let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); let trap_exit = new_trap_exit_flag(); let links = new_link_table(); let linked_reason = managed @@ -948,9 +1648,30 @@ impl ActorRef { .map(|s| s.cause.clone()) .unwrap_or_else(new_linked_exit_reason); + let id = ActorId::next(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); + let (cleaned_tx, cleaned_rx) = watch::channel(false); + // A supervised child's `ChildSpec` may override the actor's bounds. + let (max_outstanding, max_native) = managed + .as_ref() + .map(|scope| scope.bounds()) + .unwrap_or_default() + .resolve(A::MAX_OUTSTANDING, A::MAX_NATIVE_JOBS); let actor_ref = ActorRef { shared: Arc::new(ActorShared { - id: ActorId::next(), + id, + replies: Registry::new(id, max_outstanding), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, max_native)), + cleaned: cleaned_rx, sender: tx, cancellation_token: cancellation_token.clone(), completion_rx, @@ -958,6 +1679,7 @@ impl ActorRef { trap_exit, links: links.clone(), linked_reason: linked_reason.clone(), + signal, mailbox: crate::instrumentation::Mailbox::default(), managed: managed.clone(), }), @@ -966,11 +1688,15 @@ impl ActorRef { if let Some(scope) = &managed { scope.attach::(actor_ref.child_handle()); + scope.set_jobs(Arc::downgrade(&actor_ref.shared.jobs)); } + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); if let Some(parent) = parent { ctx.link(&parent); } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); #[cfg(feature = "backend-experiments")] let scheduler = if managed.is_some() { @@ -1011,41 +1737,59 @@ impl ActorRef { linked_reason, actor_ref.shared.monitors.clone(), completion_tx, - ExitReason::Kill, move || cleanup_token.cancel(), ) - .with_mailbox_failures(actor_ref.shared.sender.failures()); - let startup = - crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + .with_mailbox_failures(actor_ref.shared.sender.failures()) + .with_replies(actor_ref.shared.replies.clone()) + .with_jobs(actor_ref.shared.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); + let job_owner = actor_ref.shared.clone(); + let cleaned_tx = cleaned_tx; + // Destruction-safe: if the runtime drops this future before it + // completes, `ActorStartup` (never polled) or `ExitOnDrop` (polled) + // still reach D before destroying anything, so no caller waits for an + // instance that can never answer. let inner_future = async move { let ((actor, ctx, rx, cancellation_token), mut cleanup) = startup.into_parts(); #[cfg(feature = "backend-experiments")] let _borrowed_runtime = borrowed_runtime; #[cfg(feature = "backend-experiments")] let _borrowed_dispatcher = borrowed_dispatcher; - let reason = run_actor( + run_actor( actor, ctx, rx, cancellation_token, + &mut cleanup, #[cfg(feature = "backend-experiments")] turn, ) .await; - let reason = match reason { - Ok(reason) => { - cleanup.complete(reason.clone()); - reason - } - Err(reason) => { - cleanup.complete_secondary(reason.clone()); - reason - } - }; + let reason = cleanup.die(); if let Some(scope) = &managed { - let actual = crate::system::lock(&scope.cause).clone().unwrap_or(reason); - scope.exited(actual); + scope.exited(reason); + } + // Publication (wait_exit, monitors, link notices) after the + // state drop, before the job join. + drop(cleanup); + // Job cleanup (deferred-replies ยง5 step 4), after death: join the + // cancelled jobs, bounded by the cleanup deadline; a job still + // running then does not undo death and is reported unfinished. + if job_owner.jobs.running() > 0 { + let timeout = job_owner + .managed + .as_ref() + .map_or(DEFAULT_STOP_DEADLINE, |scope| scope.cleanup_timeout()); + let remaining = job_owner.replies.cleanup_remaining(timeout); + let unfinished = job_owner.jobs.join(remaining).await; + crate::reply::jobs::report_unfinished(job_owner.id, unfinished); } + drop(job_owner); + let _ = cleaned_tx.send(true); }; if let Some(lease) = lease { @@ -1179,17 +1923,179 @@ impl MessageTurn { } } -async fn run_actor( - mut actor: A, +/// A pending user future of the lifecycle: `started()`, a turn (handler, +/// `exit_received`, a job's `done`) or `stopped()`. If the runtime drops the +/// lifecycle while it is pending (G1), its `Drop` reaches the death point +/// first: the turn's own caller, every deferred token and every queued +/// request are answered and woken before this future and its captures (the +/// message, the handler's locals) are destroyed. A destructor panic of the +/// future is a cleanup failure. +struct DeathFirst<'a, A: Actor, C: Fn()> { + inner: Option + Send + 'a>>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + /// The run loop already polled `inner` once in this task poll (it was + /// pending): the next poll returns `Pending` without polling it again. + polled: bool, +} +impl<'a, A: Actor, C: Fn()> DeathFirst<'a, A, C> { + fn new( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + ) -> Self { + Self { + inner: Some(inner), + rx, + cleanup, + caller, + polled: false, + } + } + + /// A turn the run loop polled once and found pending. + fn pending( + inner: Pin + Send + 'a>>, + rx: &'a mut mpsc::Receiver + Send>>, + cleanup: &'a mut crate::actor_cleanup::ActorCleanup, + caller: Option<&'a dyn Caller>, + ) -> Self { + let mut turn = Self::new(inner, rx, cleanup, caller); + turn.polled = true; + turn + } +} + +/// A boxed turn of the lifecycle. +type TurnFuture<'a> = Pin + Send + 'a>>; + +/// Poll `turn` once, catching its panic. +#[inline] +fn poll_caught( + turn: &mut TurnFuture<'_>, + cx: &mut std::task::Context<'_>, +) -> Result, Panic> { + std::panic::catch_unwind(AssertUnwindSafe(|| turn.as_mut().poll(cx))) +} + +/// Drop a turn that panicked; a panic of its destructor is disposed of. +fn drop_panicked(turn: Option>) { + if let Err(secondary) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(turn))) { + crate::mailbox::dispose_panic(secondary); + } +} +/// A caught panic payload. +type Panic = Box; + +impl Future for DeathFirst<'_, A, C> { + /// The future's panic, caught: the turn's (or hook's) panic, not a + /// runtime drop. The future is gone before it is returned. + type Output = Result<(), Panic>; + fn poll( + self: Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll { + let this = self.get_mut(); + if std::mem::take(&mut this.polled) { + // Polled (and its waker registered) in this same task poll. + return std::task::Poll::Pending; + } + let inner = this.inner.as_mut().expect("polled after completion"); + match poll_caught(inner, cx) { + Ok(std::task::Poll::Pending) => std::task::Poll::Pending, + Ok(std::task::Poll::Ready(())) => { + this.inner = None; + std::task::Poll::Ready(Ok(())) + } + Err(panic) => { + drop_panicked(this.inner.take()); + std::task::Poll::Ready(Err(panic)) + } + } + } +} +impl Drop for DeathFirst<'_, A, C> { + fn drop(&mut self) { + let Some(inner) = self.inner.take() else { + return; + }; + // Destroyed while pending: the runtime dropped the actor. + self.cleanup.die_dropped(); + if let Some(caller) = self.caller.take() { + self.cleanup.answer_turn(caller.retain()); + } + let queued = answer_mailbox(self.rx, self.cleanup.replies()); + self.cleanup.discharged(); + self.rx.discard_all(queued); + if let Err(panic) = std::panic::catch_unwind(AssertUnwindSafe(|| drop(inner))) { + let message = panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + self.cleanup + .cleanup_failed(format!("dropped turn destructor: {message}")); + } + } +} + +/// The actor's state and mailbox while its lifecycle runs. They are +/// destroyed only after the death point D: normally at the end of +/// [`run_actor`]; if the lifecycle future is dropped mid-way (its runtime was +/// destroyed), here, which reaches D first (`Killed` unless a reason was +/// requested), so tokens and queued requests answer `ActorStopped` before +/// any destructor runs. +struct ExitOnDrop<'c, A: Actor, C: Fn()> { + actor: Option, + rx: mpsc::Receiver + Send>>, + cleanup: &'c mut crate::actor_cleanup::ActorCleanup, +} +impl ExitOnDrop<'_, A, C> { + /// D, then the queued requests, then destruction (mailbox, then state). + fn finish(&mut self, dropped: bool) { + let Some(actor) = self.actor.take() else { + return; + }; + if dropped { + self.cleanup.die_dropped(); + } else { + self.cleanup.die(); + } + let queued = answer_mailbox(&mut self.rx, self.cleanup.replies()); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). A destructor panic is a cleanup failure, + // never the reason. + self.cleanup.discharged(); + self.rx.discard_all(queued); + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + self.cleanup.cleanup_failed(failure); + } + } +} +impl Drop for ExitOnDrop<'_, A, C> { + fn drop(&mut self) { + self.finish(true); + } +} + +async fn run_actor( + actor: A, ctx: Context, - mut rx: mpsc::Receiver + Send>>, + rx: mpsc::Receiver + Send>>, cancellation_token: CancellationToken, + cleanup: &mut crate::actor_cleanup::ActorCleanup, #[cfg(feature = "backend-experiments")] turn: Option, -) -> Result { - let primary = match AssertUnwindSafe(run_actor_lifecycle( - &mut actor, +) { + let request = ctx.shared.linked_reason.clone(); + let mut owned = ExitOnDrop { + actor: Some(actor), + rx, + cleanup, + }; + if let Err(payload) = AssertUnwindSafe(run_actor_lifecycle( + owned.actor.as_mut().expect("the state lives until death"), ctx, - &mut rx, + &mut owned.rx, + &mut *owned.cleanup, cancellation_token, #[cfg(feature = "backend-experiments")] turn, @@ -1197,35 +2103,49 @@ async fn run_actor( .catch_unwind() .await { - Ok(reason) => Ok(reason), - Err(payload) => { - let message = panic_message(&*payload); - crate::mailbox::dispose_panic(payload); - Err(ExitReason::Panic(format!( - "actor lifecycle unwound: {message}" - ))) - } - }; - crate::actor_cleanup::finish_actor_state(actor, primary) + let message = panic_message(&*payload); + crate::mailbox::dispose_panic(payload); + link::lock_request(&request).panic(ExitReason::Panic(format!( + "actor lifecycle unwound: {message}" + ))); + } + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is answered), then + // the mailbox refuses admission and its queued requests are answered, + // all before any destructor. Early returns (startup panic, closed scope) + // and an unwound lifecycle also come here, so jobs never outlive their + // owner. + owned.finish(false); } -async fn run_actor_lifecycle( +async fn run_actor_lifecycle( actor: &mut A, ctx: Context, rx: &mut mpsc::Receiver + Send>>, + cleanup: &mut crate::actor_cleanup::ActorCleanup, cancellation_token: CancellationToken, #[cfg(feature = "backend-experiments")] mut turn: Option, -) -> ExitReason { +) { let _monitor_cleanup = MonitorCleanup(ctx.shared.monitors.clone()); + let request = ctx.shared.linked_reason.clone(); + let trace_id = ctx.shared.id.raw(); if let Some(scope) = &ctx.shared.managed { if !scope.can_start() { scope.stop(ExitReason::Shutdown); - return ExitReason::Normal; + return; } } let start_result = AssertUnwindSafe(async { fail_point!("actor.before_started", async); - actor.started(&ctx).await + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.started(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) }) .catch_unwind() .await; @@ -1234,8 +2154,21 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in started() callback: {msg}"); }); + // No stopped() after a failed start (L5). + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); + cancellation_token.cancel(); + return; + } + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.shared.trap_exit.load(Ordering::Acquire)) { cancellation_token.cancel(); - return ExitReason::Panic(format!("panic in started(): {msg}")); } fail_point!("actor.after_started_before_ready", async); @@ -1245,38 +2178,34 @@ async fn run_actor_lifecycle( } } - if cancellation_token.is_cancelled() { - if let Some(scope) = &ctx.shared.managed { - scope.end_running(); - } - if let Err(panic) = AssertUnwindSafe(async { - fail_point!("actor.before_stopped", async); - actor.stopped(&ctx).await - }) - .catch_unwind() - .await - { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - return ExitReason::Panic(format!("panic in stopped(): {msg}")); - } - return ExitReason::Normal; - } - - let mut exit_reason = ExitReason::Normal; - + // One cancellation future for the whole loop: polled only when the + // mailbox has nothing, so a busy actor registers no waiter per message. + let mut cancelled = pin!(cancellation_token.cancelled()); + // The receive checks the token before taking each message (and returns + // `None` once it is cancelled), so the loop needs no check of its own: + // `is_cancelled` locks the token's node. loop { let msg = { let recv = pin!(future::poll_fn(|cx| { // Stop and closed-mailbox both take the normal lifecycle exit, // just like the cancellation arm below. Check before consuming // work, including when receive resumes from a pending poll. - if cancellation_token.is_cancelled() { - return std::task::Poll::Ready(None); - } - let result = rx.poll_recv(cx); + // The check, the receive and its trace event are one step + // of the lifecycle trace: a recorded stop is seen here. + let result = crate::lifecycle_trace::life_section(trace_id, || { + if cancellation_token.is_cancelled() { + return std::task::Poll::Ready(None); + } + let result = rx.poll_recv(cx); + #[cfg(feature = "lifecycle-trace")] + if let std::task::Poll::Ready(Some(envelope)) = &result { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + } + result + }); #[cfg(feature = "backend-experiments")] if let Some(turn) = &mut turn { if result.is_pending() { @@ -1287,11 +2216,15 @@ async fn run_actor_lifecycle( } result })); - let cancel = pin!(cancellation_token.cancelled()); - match future::select(recv, cancel).await { - future::Either::Left((msg, _)) => msg, - future::Either::Right(_) => None, - } + let mut recv = recv; + future::poll_fn(|cx| match recv.as_mut().poll(cx) { + std::task::Poll::Pending => match cancelled.as_mut().poll(cx) { + std::task::Poll::Ready(()) => std::task::Poll::Ready(None), + std::task::Poll::Pending => std::task::Poll::Pending, + }, + ready => ready, + }) + .await }; match msg { Some(envelope) => { @@ -1311,9 +2244,46 @@ async fn run_actor_lifecycle( let observation = envelope .observation() .map(|observation| observation.begin()); - let result = AssertUnwindSafe(envelope.handle(actor, &ctx)) - .catch_unwind() - .await; + ctx.shared.replies.begin_turn(); + // The handler's future is built inside the guarded future: a + // manual `Handler` may panic before it returns one, and that + // is a handler panic like any other (threads mode catches the + // same call). The envelope outlives the turn and lends it the + // request's slot. + let mut envelope = envelope; + let result = match dispatch(&mut *envelope, &mut *actor, &ctx) { + Ok((turn, caller)) => { + let mut turn = + Some(crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), turn)); + // Most turns finish in their first poll, which runs + // here, with no await point before it returns: only + // a turn that stays pending is guarded by + // `DeathFirst` against the runtime dropping it. + let first = future::poll_fn(|cx| { + std::task::Poll::Ready(poll_caught( + turn.as_mut().expect("polled once"), + cx, + )) + }) + .await; + match first { + Ok(std::task::Poll::Ready(())) => { + drop(turn); + Ok(()) + } + Ok(std::task::Poll::Pending) => { + let turn = turn.take().expect("pending turn"); + DeathFirst::pending(turn, &mut *rx, &mut *cleanup, caller).await + } + Err(panic) => { + drop_panicked(turn.take()); + Err(panic) + } + } + } + Err(panic) => Err(panic), + }; + ctx.shared.replies.end_turn(result.is_err()); #[cfg(feature = "diagnostics")] drop(_turn); #[cfg(feature = "metrics")] @@ -1325,7 +2295,15 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in message handler: {msg}"); }); - exit_reason = ExitReason::Panic(format!("panic in handler: {msg}")); + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); break; } #[cfg(feature = "backend-experiments")] @@ -1346,9 +2324,47 @@ async fn run_actor_lifecycle( scope.end_running(); } cancellation_token.cancel(); + ctx.shared.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.shared.sender.failures().hold(); + ctx.shared.jobs.cancel_all(); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) + }); + (run, skipped) + }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.shared.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } + if !run_hook { + return; + } let stop_result = AssertUnwindSafe(async { fail_point!("actor.before_stopped", async); - actor.stopped(&ctx).await + DeathFirst::new( + crate::lifecycle_trace::in_actor(ctx.shared.id.raw(), Box::pin(actor.stopped(&ctx))), + &mut *rx, + &mut *cleanup, + None, + ) + .await + .unwrap_or_else(|panic| std::panic::resume_unwind(panic)) }) .catch_unwind() .await; @@ -1357,12 +2373,52 @@ async fn run_actor_lifecycle( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in stopped() callback: {msg}"); }); - if !exit_reason.is_abnormal() { - exit_reason = ExitReason::Panic(format!("panic in stopped(): {msg}")); - } + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); } +} - exit_reason +/// A context and its cleanup without spawning, for run-loop unit tests. +#[cfg(test)] +fn test_context( + tx: &mpsc::Sender + Send>>, + token: &CancellationToken, +) -> (Context, crate::actor_cleanup::ActorCleanup) { + let id = ActorId::next(); + let (completion_tx, completion_rx) = watch::channel(None); + let trap = new_trap_exit_flag(); + let links = new_link_table(); + let request = new_linked_exit_reason(); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); + let signal = signal_target::(id, None, tx, trap.clone(), request.clone(), token.clone()); + let ctx = Context { + shared: Arc::new(ActorShared { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion_rx, + monitors: monitors.clone(), + trap_exit: trap, + links: links.clone(), + linked_reason: request.clone(), + signal, + mailbox: crate::instrumentation::Mailbox::default(), + managed: None, + replies: Registry::new(id, DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(id, 1)), + cleaned: watch::channel(false).1, + }), + }; + let noop: fn() = || {}; + let cleanup = + crate::actor_cleanup::ActorCleanup::new(id, links, request, monitors, completion_tx, noop) + .with_replies(ctx.shared.replies.clone()); + (ctx, cleanup) } // --------------------------------------------------------------------------- @@ -1389,9 +2445,17 @@ pub trait ActorStart: Actor { ActorRef::spawn(self, backend, None) } - /// Register the bidirectional link before the child can run `started()`. - /// A closed parent is observed under its link-state mutex; an abnormal - /// parent exit cancels the child cooperatively before message dispatch. + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. A + /// parent already past its death point gives the child a `NoProc` parent + /// exit. + /// + /// The child is always an unmanaged `Backend::Async` actor, even when + /// the parent is managed: it is outside the parent's execution group and + /// its ordered shutdown (`spawn_helper` on it returns `UnmanagedActor`). + /// The unified start API (planned for 0.9) will inherit the placement. fn start_linked(self, parent_ctx: &Context

) -> ActorRef { ActorRef::spawn(self, Backend::Async, Some(parent_ctx.child_handle())) } @@ -1516,22 +2580,43 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { - self.count + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); } } impl Handler for Counter { - async fn handle(&mut self, _msg: Increment, _ctx: &Context) -> u64 { - self.count += 1; - self.count + async fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { + self.count += 1; + self.count + }; + reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { - ctx.stop(); - self.count + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); } } @@ -1569,141 +2654,180 @@ mod tests { #[test] pub fn backend_async_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); + crate::test_wait::bounded( + concat!(module_path!(), "::backend_async_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - // fire-and-forget send; the request below shares its mailbox, so - // it is handled after the send without any extra wait. - counter.send(Increment).unwrap(); + // fire-and-forget send; the request below shares its mailbox, so + // it is handled after the send without any extra wait. + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn backend_blocking_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); + crate::test_wait::bounded( + concat!( + module_path!(), + "::backend_blocking_handles_send_and_request" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Blocking); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - counter.send(Increment).unwrap(); + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn backend_thread_handles_send_and_request() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); + crate::test_wait::bounded( + concat!(module_path!(), "::backend_thread_handles_send_and_request"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start_with_backend(Backend::Thread); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 0); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 0); - let result = counter.request(Increment).await.unwrap(); - assert_eq!(result, 1); + let result = counter.request(Increment).await.unwrap(); + assert_eq!(result, 1); - counter.send(Increment).unwrap(); + counter.send(Increment).unwrap(); - let result = counter.request(GetCount).await.unwrap(); - assert_eq!(result, 2); + let result = counter.request(GetCount).await.unwrap(); + assert_eq!(result, 2); - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 2); - }); + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 2); + }); + }, + ); } #[test] pub fn multiple_backends_concurrent() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let async_counter = Counter { count: 0 }.start(); - let blocking_counter = Counter { count: 100 }.start_with_backend(Backend::Blocking); - let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); - - async_counter.request(Increment).await.unwrap(); - blocking_counter.request(Increment).await.unwrap(); - thread_counter.request(Increment).await.unwrap(); - - let async_val = async_counter.request(GetCount).await.unwrap(); - let blocking_val = blocking_counter.request(GetCount).await.unwrap(); - let thread_val = thread_counter.request(GetCount).await.unwrap(); - - assert_eq!(async_val, 1); - assert_eq!(blocking_val, 101); - assert_eq!(thread_val, 201); - - async_counter.request(StopCounter).await.unwrap(); - blocking_counter.request(StopCounter).await.unwrap(); - thread_counter.request(StopCounter).await.unwrap(); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_backends_concurrent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let async_counter = Counter { count: 0 }.start(); + let blocking_counter = + Counter { count: 100 }.start_with_backend(Backend::Blocking); + let thread_counter = Counter { count: 200 }.start_with_backend(Backend::Thread); + + async_counter.request(Increment).await.unwrap(); + blocking_counter.request(Increment).await.unwrap(); + thread_counter.request(Increment).await.unwrap(); + + let async_val = async_counter.request(GetCount).await.unwrap(); + let blocking_val = blocking_counter.request(GetCount).await.unwrap(); + let thread_val = thread_counter.request(GetCount).await.unwrap(); + + assert_eq!(async_val, 1); + assert_eq!(blocking_val, 101); + assert_eq!(thread_val, 201); + + async_counter.request(StopCounter).await.unwrap(); + blocking_counter.request(StopCounter).await.unwrap(); + thread_counter.request(StopCounter).await.unwrap(); + }); + }, + ); } #[test] pub fn request_timeout() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct SlowActor; - struct SlowOp; - impl Message for SlowOp { - type Result = (); - } - impl Actor for SlowActor {} - impl Handler for SlowActor { - async fn handle(&mut self, _msg: SlowOp, _ctx: &Context) { - // Never replies, so the timeout is the only possible outcome - // however late the runtime polls it. - std::future::pending::<()>().await; + crate::test_wait::bounded(concat!(module_path!(), "::request_timeout"), move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct SlowActor; + struct SlowOp; + impl Message for SlowOp { + type Result = (); + } + impl Actor for SlowActor {} + impl Handler for SlowActor { + async fn handle( + &mut self, + _msg: SlowOp, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + // Never replies, so the timeout is the only possible outcome + // however late the runtime polls it. + std::future::pending::<()>().await; + reply.reply(()); + } } - } - let actor = SlowActor.start(); - let result = actor - .request_with_timeout(SlowOp, Duration::from_millis(50)) - .await; - assert!(matches!(result, Err(ActorError::RequestTimeout))); + let actor = SlowActor.start(); + let result = actor + .request_with_timeout(SlowOp, Duration::from_millis(50)) + .await; + assert!(matches!(result, Err(ActorError::RequestTimeout))); + }); }); } #[test] pub fn recipient_type_erasure() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 42 }.start(); - let recipient: Recipient = counter.recipient(); - - let rx = recipient.request_raw(GetCount).unwrap(); - let result = rx.await.unwrap(); - assert_eq!(result, 42); - - // Also test request helper - let result = request(&*recipient, GetCount, Duration::from_secs(5)) - .await - .unwrap(); - assert_eq!(result, 42); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 42 }.start(); + let recipient: Recipient = counter.recipient(); + + let rx = recipient + .send_request(GetCount, Deadline::after(DEFAULT_REQUEST_TIMEOUT)) + .unwrap(); + let result = rx.await.unwrap(); + assert_eq!(result, 42); + + // Also test request helper + let result = request(&*recipient, GetCount, Duration::from_secs(5)) + .await + .unwrap(); + assert_eq!(result, 42); + }); + }, + ); } // --- SlowShutdownActor for join tests --- @@ -1722,8 +2846,14 @@ mod tests { } impl Handler for SlowShutdownActor { - async fn handle(&mut self, _msg: StopSlow, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop(); + reply.reply(()); } } @@ -1762,73 +2892,98 @@ mod tests { } impl Handler for HeldThread { - async fn handle(&mut self, _msg: StopSlow, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop(); + reply.reply(()); } } impl Handler for HeldThread { - async fn handle(&mut self, _msg: DoBlock, ctx: &Context) { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { self.hold(); ctx.stop(); + reply.reply(()); } } #[test] pub fn thread_backend_join_does_not_block_runtime() { - // stopped() blocks its OS thread until a task on this current-thread - // runtime releases it. If join() blocked the runtime, that task could - // never run and stopped() would give up after the timeout instead. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::thread_backend_join_does_not_block_runtime" + ), + move || { + // stopped() blocks its OS thread until a task on this current-thread + // runtime releases it. If join() blocked the runtime, that task could + // never run and stopped() would give up after the timeout instead. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); - runtime.block_on(async move { - let (held, release) = HeldThread::new(); - let released = held.released.clone(); - let actor = held.start_with_backend(Backend::Thread); - actor.send(StopSlow).unwrap(); - let _releaser = rt::spawn(async move { - let _ = release.send(()); - }); + runtime.block_on(async move { + let (held, release) = HeldThread::new(); + let released = held.released.clone(); + let actor = held.start_with_backend(Backend::Thread); + actor.send(StopSlow).unwrap(); + let _releaser = rt::spawn(async move { + let _ = release.send(()); + }); - actor.join().await; + actor.join().await; - assert!( - released.load(atomic::Ordering::SeqCst), - "join() blocked the runtime: the releasing task never ran" - ); - }); + assert!( + released.load(atomic::Ordering::SeqCst), + "join() blocked the runtime: the releasing task never ran" + ); + }); + }, + ); } #[test] pub fn multiple_join_callers_all_notified() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = SlowShutdownActor.start(); - let actor_clone1 = actor.clone(); - let actor_clone2 = actor.clone(); - - let join1 = rt::spawn(async move { - actor_clone1.join().await; - 1u32 - }); - let join2 = rt::spawn(async move { - actor_clone2.join().await; - 2u32 - }); + crate::test_wait::bounded( + concat!(module_path!(), "::multiple_join_callers_all_notified"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = SlowShutdownActor.start(); + let actor_clone1 = actor.clone(); + let actor_clone2 = actor.clone(); + + let join1 = rt::spawn(async move { + actor_clone1.join().await; + 1u32 + }); + let join2 = rt::spawn(async move { + actor_clone2.join().await; + 2u32 + }); - rt::sleep(Duration::from_millis(10)).await; + rt::sleep(Duration::from_millis(10)).await; - actor.send(StopSlow).unwrap(); + actor.send(StopSlow).unwrap(); - let (r1, r2) = tokio::join!(join1, join2); - assert_eq!(r1.unwrap(), 1); - assert_eq!(r2.unwrap(), 2); + let (r1, r2) = tokio::join!(join1, join2); + assert_eq!(r1.unwrap(), 1); + assert_eq!(r2.unwrap(), 2); - actor.join().await; - }); + actor.join().await; + }); + }, + ); } // --- Badly behaved actors for blocking tests --- @@ -1843,10 +2998,16 @@ mod tests { impl Actor for BadlyBehavedTask {} impl Handler for BadlyBehavedTask { - async fn handle(&mut self, _msg: DoBlock, ctx: &Context) { + async fn handle( + &mut self, + _msg: DoBlock, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { rt::sleep(Duration::from_millis(20)).await; thread::sleep(Duration::from_secs(2)); ctx.stop(); + reply.reply(()); } } @@ -1862,47 +3023,71 @@ mod tests { impl Actor for WellBehavedTask {} impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { - self.count + async fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { - ctx.stop(); - self.count + async fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); } } impl Handler for WellBehavedTask { - async fn handle(&mut self, _msg: IncrementWell, ctx: &Context) { + async fn handle( + &mut self, + _msg: IncrementWell, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { self.count += 1; use crate::tasks::send_after; send_after(Duration::from_millis(100), ctx.clone(), IncrementWell); + reply.reply(()); } } #[test] pub fn badly_behaved_thread_non_blocking() { - // An async actor that blocks its thread starves everything else on - // it. With one runtime thread this is deterministic; on a multi-thread - // runtime the well-behaved actor may run on another worker. - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - runtime.block_on(async move { - let badboy = BadlyBehavedTask.start(); - badboy.send(DoBlock).unwrap(); - let goodboy = WellBehavedTask { count: 0 }.start(); - goodboy.send(IncrementWell).unwrap(); - rt::sleep(Duration::from_secs(1)).await; - let count = goodboy.request(GetCount).await.unwrap(); - // The blocking handler held the only thread for 2 s of the 1 s - // window, so the 100 ms chain could not get far. - assert!(count < 10, "blocked runtime still made progress: {count}"); - goodboy.request(StopCounter).await.unwrap(); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::badly_behaved_thread_non_blocking"), + move || { + // An async actor that blocks its thread starves everything else on + // it. With one runtime thread this is deterministic; on a multi-thread + // runtime the well-behaved actor may run on another worker. + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime.block_on(async move { + let badboy = BadlyBehavedTask.start(); + badboy.send(DoBlock).unwrap(); + let goodboy = WellBehavedTask { count: 0 }.start(); + goodboy.send(IncrementWell).unwrap(); + rt::sleep(Duration::from_secs(1)).await; + let count = goodboy.request(GetCount).await.unwrap(); + // The blocking handler held the only thread for 2 s of the 1 s + // window, so the 100 ms chain could not get far. + assert!(count < 10, "blocked runtime still made progress: {count}"); + goodboy.request(StopCounter).await.unwrap(); + }); + }, + ); } /// An async actor on a single-threaded runtime keeps making progress while @@ -1955,174 +3140,311 @@ mod tests { #[test] pub fn panic_in_started_stops_actor() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicOnStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Ping, _ctx: &Context) {} - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicOnStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicOnStart.start(); - actor.join().await; - let result = actor.send(Ping); - assert!(result.is_err()); - }); + let actor = PanicOnStart.start(); + actor.join().await; + let result = actor.send(Ping); + assert!(result.is_err()); + }); + }, + ); } #[test] pub fn panic_in_handler_stops_actor() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnMsg; - struct Explode; - impl Message for Explode { - type Result = (); - } - struct Check; - impl Message for Check { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Explode, _ctx: &Context) { - panic!("boom in handler"); - } - } - impl Handler for PanicOnMsg { - async fn handle(&mut self, _msg: Check, _ctx: &Context) -> u32 { - 42 - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnMsg; + struct Explode; + impl Message for Explode { + type Result = (); + } + struct Check; + impl Message for Check { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + async fn handle( + &mut self, + _msg: Check, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } + } - let actor = PanicOnMsg.start(); - actor.send(Explode).unwrap(); - actor.join().await; - let result = actor.request(Check).await; - assert!(result.is_err()); - }); + let actor = PanicOnMsg.start(); + actor.send(Explode).unwrap(); + actor.join().await; + let result = actor.request(Check).await; + assert!(result.is_err()); + }); + }, + ); } + /// Review P1 (Codex 9): a manual `Handler` may be a plain `fn` that + /// panics before it returns its future. That is a handler panic like + /// any other: the turn ends, `stopped()` runs, and the caller and + /// `wait_exit` see the handler-panic reason. The future used to be built + /// outside the turn's `catch_unwind`, so the panic unwound the whole + /// lifecycle (no `stopped()`, an "actor lifecycle unwound" reason). #[test] - pub fn panic_in_stopped_still_completes() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - struct PanicOnStop; - struct StopMe; - impl Message for StopMe { - type Result = (); - } - impl Actor for PanicOnStop { - async fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - async fn handle(&mut self, _msg: StopMe, ctx: &Context) { - ctx.stop(); - } - } + pub fn panic_building_the_handler_future_is_a_handler_panic() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::panic_building_the_handler_future_is_a_handler_panic" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct Eager(Arc); + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for Eager { + async fn stopped(&mut self, _ctx: &Context) { + self.0.store(true, Ordering::SeqCst); + } + } + impl Handler for Eager { + #[allow(unreachable_code, clippy::diverging_sub_expression)] + fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) -> impl Future + Send { + panic!("boom building the handler future"); + async {} + } + } - let actor = PanicOnStop.start(); - actor.send(StopMe).unwrap(); - actor.join().await; - }); - } + let stopped = Arc::new(atomic::AtomicBool::new(false)); + let actor = Eager(stopped.clone()).start(); + let reply = actor.request(Explode).await; + let reason = actor.wait_exit().await; + let ExitReason::Panic(message) = &reason else { + panic!("unexpected reason {reason:?}"); + }; + assert!( + message.starts_with("panic in handler: boom building"), + "not a handler panic: {message}" + ); + assert!( + matches!(&reply, Err(ActorError::ActorStopped(r)) if *r == reason), + "caller saw {reply:?}" + ); + assert!(stopped.load(Ordering::SeqCst), "stopped() was skipped"); + }); + }, + ); + } + + #[test] + pub fn panic_in_stopped_still_completes() { + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + struct PanicOnStop; + struct StopMe; + impl Message for StopMe { + type Result = (); + } + impl Actor for PanicOnStop { + async fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + async fn handle( + &mut self, + _msg: StopMe, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } + + let actor = PanicOnStop.start(); + actor.send(StopMe).unwrap(); + actor.join().await; + }); + }, + ); + } #[test] pub fn send_message_on_delivers() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); - let count = crate::test_wait::wait_for_async( - || counter.request(GetCount), - |count| matches!(count, Ok(1)), - ) - .await - .unwrap(); - assert_eq!(count, 1); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(10)), Increment); + let count = crate::test_wait::wait_for_async( + || counter.request(GetCount), + |count| matches!(count, Ok(1)), + ) + .await + .unwrap(); + assert_eq!(count, 1); + }); + }, + ); } #[test] pub fn send_message_on_cancelled() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let counter = Counter { count: 0 }.start(); - let ctx = counter.context(); - send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); - // Stop actor before the future resolves - let final_count = counter.request(StopCounter).await.unwrap(); - assert_eq!(final_count, 0, "message should not have been delivered"); - counter.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_cancelled"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let counter = Counter { count: 0 }.start(); + let ctx = counter.context(); + send_message_on(ctx, rt::sleep(Duration::from_millis(200)), Increment); + // Stop actor before the future resolves + let final_count = counter.request(StopCounter).await.unwrap(); + assert_eq!(final_count, 0, "message should not have been delivered"); + counter.join().await; + }); + }, + ); } // --- ExitReason tests --- #[test] pub fn exit_reason_normal_on_clean_stop() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).await.unwrap(); - let reason = actor.wait_exit().await; - assert!(matches!(reason, ExitReason::Normal)); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).await.unwrap(); + let reason = actor.wait_exit().await; + assert!(matches!(reason, ExitReason::Normal)); + }); + }, + ); } #[test] pub fn exit_reason_panic_in_started() { - struct PanicStart; - struct Ping; - impl Message for Ping { - type Result = (); - } - impl Actor for PanicStart { - async fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStart { - async fn handle(&mut self, _msg: Ping, _ctx: &Context) {} - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStart; + struct Ping; + impl Message for Ping { + type Result = (); + } + impl Actor for PanicStart { + async fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStart { + async fn handle( + &mut self, + _msg: Ping, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { let actor = PanicStart.start(); let reason = actor.wait_exit().await; assert!( matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) ); }); + }, + ); } #[test] pub fn exit_reason_panic_in_handler() { - struct PanicHandler; - struct Explode; - impl Message for Explode { - type Result = (); - } - impl Actor for PanicHandler {} - impl Handler for PanicHandler { - async fn handle(&mut self, _msg: Explode, _ctx: &Context) { - panic!("boom in handler"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandler; + struct Explode; + impl Message for Explode { + type Result = (); + } + impl Actor for PanicHandler {} + impl Handler for PanicHandler { + async fn handle( + &mut self, + _msg: Explode, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { let actor = PanicHandler.start(); let _ = actor.send(Explode); let reason = actor.wait_exit().await; @@ -2130,18 +3452,28 @@ mod tests { matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) ); }); + }, + ); } #[test] pub fn exit_reason_poll_returns_none_while_running() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).await.unwrap(); - actor.join().await; - assert!(actor.exit_reason().is_some()); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).await.unwrap(); + actor.join().await; + assert!(actor.exit_reason().is_some()); + }); + }, + ); } // --- Monitor tests --- @@ -2165,6 +3497,11 @@ mod tests { impl Message for CallDemonitor { type Result = (); } + /// Monitor, then demonitor in the same turn. + struct MonitorThenDemonitor(crate::ChildHandle); + impl Message for MonitorThenDemonitor { + type Result = (); + } impl Actor for Watcher {} @@ -2172,21 +3509,47 @@ mod tests { async fn handle( &mut self, msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, ctx: &Context, - ) -> crate::monitor::MonitorRef { - ctx.monitor(&msg.0) + ) { + reply.reply(ctx.monitor(&msg.0)); + } + } + + impl Handler for Watcher { + async fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: CallDemonitor, ctx: &Context) { + async fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Watcher { - async fn handle(&mut self, msg: crate::monitor::Down, _ctx: &Context) { + async fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.downs.lock().unwrap().push(msg); + reply.reply(()); } } @@ -2194,243 +3557,340 @@ mod tests { async fn handle( &mut self, _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, _ctx: &Context, - ) -> Vec { - self.downs.lock().unwrap().clone() + ) { + reply.reply(self.downs.lock().unwrap().clone()); } } #[test] pub fn monitor_running_actor_delivers_down_on_exit() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + // Stop the target โ€” Down should be delivered + target.request(StopCounter).await.unwrap(); + target.join().await; - // Stop the target โ€” Down should be delivered - target.request(StopCounter).await.unwrap(); - target.join().await; + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }); + }, + ); + } - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); - }); + /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox + /// (here the `NoProc` a monitor of a dead target queues at once) is + /// never handled once `demonitor` returned. + #[test] + pub fn demonitor_flushes_a_queued_down() { + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .await + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }); + }, + ); } #[test] pub fn monitor_already_dead_actor_delivers_down_immediately() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).await.unwrap(); - target.join().await; - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).await.unwrap(); + target.join().await; + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - }); + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + }); + }, + ); } #[test] pub fn demonitor_before_target_dies_suppresses_down() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); - watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); + let monitor_ref = watcher.request(StartMonitor(target_handle)).await.unwrap(); + watcher.request(CallDemonitor(monitor_ref)).await.unwrap(); - // Now stop the target - target.request(StopCounter).await.unwrap(); - target.join().await; - rt::sleep(Duration::from_millis(50)).await; + // Now stop the target + target.request(StopCounter).await.unwrap(); + target.join().await; + rt::sleep(Duration::from_millis(50)).await; - let downs = watcher.request(GetDowns).await.unwrap(); - assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); - }); + let downs = watcher.request(GetDowns).await.unwrap(); + assert!(downs.is_empty(), "expected no Down, got {:?}", downs.len()); + }); + }, + ); } #[test] pub fn multiple_monitors_each_get_own_ref_and_down() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .await - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .await + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).await.unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }); + }, + ); } #[test] pub fn monitor_table_is_cleaned_up_after_target_dies() { - // Watcher should remove its entry from the monitor table after the - // target dies, so the table doesn't accumulate stale entries. - struct Inspect; - impl Message for Inspect { - type Result = usize; - } - impl Handler for Watcher { - async fn handle(&mut self, _msg: Inspect, ctx: &Context) -> usize { - ctx.shared - .monitors - .lock() - .unwrap() - .as_ref() - .map_or(0, HashMap::len) - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_table_is_cleaned_up_after_target_dies" + ), + move || { + // Watcher should remove its entry from the monitor table after the + // target dies, so the table doesn't accumulate stale entries. + struct Inspect; + impl Message for Inspect { + type Result = usize; + } + impl Handler for Watcher { + async fn handle( + &mut self, + _msg: Inspect, + reply: crate::reply::PendingReply<'_, usize>, + ctx: &Context, + ) { + reply.reply( + ctx.shared + .monitors + .lock() + .unwrap() + .as_ref() + .map_or(0, HashMap::len), + ); + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); - - let _ = watcher - .request(StartMonitor(target.child_handle())) - .await - .unwrap(); - assert_eq!(watcher.request(Inspect).await.unwrap(), 1); - - target.request(StopCounter).await.unwrap(); - target.join().await; - let entries = crate::test_wait::wait_for_async( - || watcher.request(Inspect), - |entries| !matches!(entries, Ok(n) if *n > 0), - ) - .await; - assert_eq!( - entries.unwrap(), - 0, - "monitor table should be empty after target died" - ); - }); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); + + let _ = watcher + .request(StartMonitor(target.child_handle())) + .await + .unwrap(); + assert_eq!(watcher.request(Inspect).await.unwrap(), 1); + + target.request(StopCounter).await.unwrap(); + target.join().await; + let entries = crate::test_wait::wait_for_async( + || watcher.request(Inspect), + |entries| !matches!(entries, Ok(n) if *n > 0), + ) + .await; + assert_eq!( + entries.unwrap(), + 0, + "monitor table should be empty after target died" + ); + }); + }, + ); } #[test] pub fn monitor_observes_panic_reason() { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - async fn handle(&mut self, _msg: PanicMsg, _ctx: &Context) { - panic!("intentional panic"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + async fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = PanicMe.start(); - let target_handle = target.child_handle(); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = PanicMe.start(); + let target_handle = target.child_handle(); - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - let _ = target.send(PanicMsg); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = target.send(PanicMsg); - let downs = crate::test_wait::wait_for_async( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .await - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); - }); + let downs = crate::test_wait::wait_for_async( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .await + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }); + }, + ); } #[test] pub fn monitoring_actor_stops_before_target_does_not_panic() { - // Regression: if the monitoring actor stops while the target is still - // alive, the watcher must not crash when the target eventually dies. - // The watcher's send() will fail silently (mailbox closed), and the - // watcher exits cleanly. - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - - let watcher = Watcher { - downs: Arc::new(Mutex::new(Vec::new())), - } - .start(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitoring_actor_stops_before_target_does_not_panic" + ), + move || { + // Regression: if the monitoring actor stops while the target is still + // alive, the watcher must not crash when the target eventually dies. + // The watcher's send() will fail silently (mailbox closed), and the + // watcher exits cleanly. + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + + let watcher = Watcher { + downs: Arc::new(Mutex::new(Vec::new())), + } + .start(); - let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); + let _ = watcher.request(StartMonitor(target_handle)).await.unwrap(); - // Stop the monitoring actor first - let watcher_handle = watcher.child_handle(); - watcher_handle.stop(); - watcher_handle.wait_exit_async().await; + // Stop the monitoring actor first + let watcher_handle = watcher.child_handle(); + watcher_handle.stop(); + watcher_handle.wait_exit_async().await; - // Now stop the target โ€” the watcher should clean up without panicking - target.request(StopCounter).await.unwrap(); - target.join().await; - // Give the orphaned watcher task time to attempt delivery - rt::sleep(Duration::from_millis(50)).await; - // If we got here without panicking, the test passes. - }); + // Now stop the target โ€” the watcher should clean up without panicking + target.request(StopCounter).await.unwrap(); + target.join().await; + // Give the orphaned watcher task time to attempt delivery + rt::sleep(Duration::from_millis(50)).await; + // If we got here without panicking, the test passes. + }); + }, + ); } // --- Link tests --- @@ -2460,14 +3920,25 @@ mod tests { } impl Handler for TrapActor { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for TrapActor { - async fn handle(&mut self, _msg: GetExits, _ctx: &Context) -> Vec { - self.exits.lock().unwrap().clone() + async fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.exits.lock().unwrap().clone()); } } @@ -2481,341 +3952,461 @@ mod tests { #[test] pub fn link_propagates_panic_to_non_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Two non-trapping actors. When A panics, B should be terminated. - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).await.unwrap(); - - // Make A panic by dropping its only ActorRef while it's mid-handler... easier: - // make A panic via a panic message - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom, _ctx: &Context) { - panic!("boom"); - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Two non-trapping actors. When A panics, B should be terminated. + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).await.unwrap(); + + // Make A panic by dropping its only ActorRef while it's mid-handler... easier: + // make A panic via a panic message + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } - let _ = a.send(Boom); - // Both should die - let reason_a = a.wait_exit().await; - let reason_b = b.wait_exit().await; - assert!(matches!(reason_a, ExitReason::Panic(_))); - // B should propagate A's reason (transitive propagation) - assert!(matches!(reason_b, ExitReason::Panic(_))); - }); + let _ = a.send(Boom); + // Both should die + let reason_a = a.wait_exit().await; + let reason_b = b.wait_exit().await; + assert!(matches!(reason_a, ExitReason::Panic(_))); + // B should propagate A's reason (transitive propagation) + assert!(matches!(reason_b, ExitReason::Panic(_))); + }); + }, + ); } #[test] pub fn link_delivers_exit_to_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = make_trapper(false); - let b = make_trapper(true); // B traps - b.request(LinkTo(a.child_handle())).await.unwrap(); - - struct Boom2; - impl Message for Boom2 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom2, _ctx: &Context) { - panic!("boom2"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::link_delivers_exit_to_trapping_peer"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = make_trapper(false); + let b = make_trapper(true); // B traps + b.request(LinkTo(a.child_handle())).await.unwrap(); + + struct Boom2; + impl Message for Boom2 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom2"); + } + } + } - let _ = a.send(Boom2); - a.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); + let _ = a.send(Boom2); + a.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); - // B should still be alive and have received an Exit - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + // B should still be alive and have received an Exit + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); - // Clean up - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + // Clean up + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn link_normal_exit_not_propagated_to_non_trapping_peer() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(false); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // A stops cleanly - a.request(StopCounter).await.unwrap(); - a.join().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should still be alive - assert!(b.exit_reason().is_none()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_normal_exit_not_propagated_to_non_trapping_peer" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(false); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // A stops cleanly + a.request(StopCounter).await.unwrap(); + a.join().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should still be alive + assert!(b.exit_reason().is_none()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] - pub fn link_to_already_dead_actor_delivers_signal() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // Start and panic a target - struct Boom3; - impl Message for Boom3 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom3, _ctx: &Context) { - panic!("boom3"); - } - } - let target = make_trapper(false); - let _ = target.send(Boom3); - target.wait_exit().await; - - // Now link from a trapping observer โ€” should receive Exit immediately - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); - - let exits = crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + pub fn link_to_already_dead_actor_delivers_noproc() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_actor_delivers_noproc" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // Start and panic a target + struct Boom3; + impl Message for Boom3 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom3, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom3"); + } + } + } + let target = make_trapper(false); + let _ = target.send(Boom3); + target.wait_exit().await; + + // Now link from a trapping observer: the target is past its + // death point, so the Exit carries NoProc (E4), not its reason. + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + let exits = crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].reason, ExitReason::NoProc); + assert!(matches!(target.exit_reason(), Some(ExitReason::Panic(_)))); - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn unlink_prevents_signal_delivery() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - - // Now unlink - struct UnlinkFrom(crate::ChildHandle); - impl Message for UnlinkFrom { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, msg: UnlinkFrom, ctx: &Context) { - ctx.unlink(&msg.0); - } - } - b.request(UnlinkFrom(a.child_handle())).await.unwrap(); + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + + // Now unlink + struct UnlinkFrom(crate::ChildHandle); + impl Message for UnlinkFrom { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + msg: UnlinkFrom, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.unlink(&msg.0); + reply.reply(()); + } + } + b.request(UnlinkFrom(a.child_handle())).await.unwrap(); - // A panics - struct Boom4; - impl Message for Boom4 { - type Result = u64; - } - impl Handler for Counter { - async fn handle(&mut self, _msg: Boom4, _ctx: &Context) -> u64 { - panic!("boom4"); - } - } - // Counter's Boom4 returns u64; just use send with no expectation of reply - let _ = a.send(Boom4); - a.wait_exit().await; - rt::sleep(Duration::from_millis(50)).await; - - // B should NOT have received an Exit - let exits = b.request(GetExits).await.unwrap(); - assert!(exits.is_empty()); - - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + // A panics + struct Boom4; + impl Message for Boom4 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom4, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom4"); + } + } + // Counter's Boom4 returns u64; just use send with no expectation of reply + let _ = a.send(Boom4); + a.wait_exit().await; + rt::sleep(Duration::from_millis(50)).await; + + // B should NOT have received an Exit + let exits = b.request(GetExits).await.unwrap(); + assert!(exits.is_empty()); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn duplicate_link_is_idempotent() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let a = Counter { count: 0 }.start(); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).await.unwrap(); - b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate - - struct Boom5; - impl Message for Boom5 { - type Result = u64; - } - impl Handler for Counter { - async fn handle(&mut self, _msg: Boom5, _ctx: &Context) -> u64 { - panic!("boom5"); - } - } - let _ = a.send(Boom5); - a.wait_exit().await; - crate::test_wait::wait_for_async( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; + crate::test_wait::bounded( + concat!(module_path!(), "::duplicate_link_is_idempotent"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let a = Counter { count: 0 }.start(); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).await.unwrap(); + b.request(LinkTo(a.child_handle())).await.unwrap(); // duplicate + + struct Boom5; + impl Message for Boom5 { + type Result = u64; + } + impl Handler for Counter { + async fn handle( + &mut self, + _msg: Boom5, + _reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + panic!("boom5"); + } + } + let _ = a.send(Boom5); + a.wait_exit().await; + crate::test_wait::wait_for_async( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; - // Should receive only ONE Exit, not two - let exits = b.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1); + // Should receive only ONE Exit, not two + let exits = b.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1); - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_async().await; - }); + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn chain_propagation_through_non_trapping_middle() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - // A linked to B linked to C. A panics โ†’ B dies (not trapping) โ†’ - // B's death propagates to C. - let a = make_trapper(false); - let b = make_trapper(false); - let c = make_trapper(true); // C traps - - a.request(LinkTo(b.child_handle())).await.unwrap(); - c.request(LinkTo(b.child_handle())).await.unwrap(); - - struct Boom6; - impl Message for Boom6 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom6, _ctx: &Context) { - panic!("boom6"); - } - } - let _ = a.send(Boom6); - a.wait_exit().await; - b.wait_exit().await; - let exits = crate::test_wait::wait_for_async( - || c.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); + crate::test_wait::bounded( + concat!( + module_path!(), + "::chain_propagation_through_non_trapping_middle" + ), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + // A linked to B linked to C. A panics โ†’ B dies (not trapping) โ†’ + // B's death propagates to C. + let a = make_trapper(false); + let b = make_trapper(false); + let c = make_trapper(true); // C traps + + a.request(LinkTo(b.child_handle())).await.unwrap(); + c.request(LinkTo(b.child_handle())).await.unwrap(); + + struct Boom6; + impl Message for Boom6 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom6, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom6"); + } + } + } + let _ = a.send(Boom6); + a.wait_exit().await; + b.wait_exit().await; + let exits = crate::test_wait::wait_for_async( + || c.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + assert_eq!(exits.len(), 1, "expected C to receive Exit from B's death"); - let ch = c.child_handle(); - ch.stop(); - ch.wait_exit_async().await; - }); + let ch = c.child_handle(); + ch.stop(); + ch.wait_exit_async().await; + }); + }, + ); } #[test] pub fn link_to_already_dead_delivers_exactly_once_to_trapping_peer() { - // Regression: previously, `ctx.link()` would deliver a duplicate Exit - // to trapping actors when the target died concurrently โ€” once from - // propagate_exit (since register_link inserted self in target's table - // before target's drain ran) and once from the fallback exit_reason() - // check. - struct Boom7; - impl Message for Boom7 { - type Result = (); - } - impl Handler for TrapActor { - async fn handle(&mut self, _msg: Boom7, _ctx: &Context) { - panic!("boom7"); - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_to_already_dead_delivers_exactly_once_to_trapping_peer" + ), + move || { + // Regression: previously, `ctx.link()` would deliver a duplicate Exit + // to trapping actors when the target died concurrently โ€” once from + // propagate_exit (since register_link inserted self in target's table + // before target's drain ran) and once from the fallback exit_reason() + // check. + struct Boom7; + impl Message for Boom7 { + type Result = (); + } + impl Handler for TrapActor { + async fn handle( + &mut self, + _msg: Boom7, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom7"); + } + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let target = make_trapper(false); - // Kick off the panic - let _ = target.send(Boom7); - // Wait for target to fully die so exit_reason() is Some - target.wait_exit().await; - - // Now link from a trapping observer โ€” should receive Exit EXACTLY ONCE - let observer = make_trapper(true); - observer - .request(LinkTo(target.child_handle())) - .await - .unwrap(); - - crate::test_wait::wait_for_async( - || observer.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .await - .unwrap(); - // Leave room for a duplicate before checking there is none. - rt::sleep(Duration::from_millis(50)).await; - let exits = observer.request(GetExits).await.unwrap(); - assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); - - let oh = observer.child_handle(); - oh.stop(); - oh.wait_exit_async().await; - }); + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let target = make_trapper(false); + // Kick off the panic + let _ = target.send(Boom7); + // Wait for target to fully die so exit_reason() is Some + target.wait_exit().await; + + // Now link from a trapping observer โ€” should receive Exit EXACTLY ONCE + let observer = make_trapper(true); + observer + .request(LinkTo(target.child_handle())) + .await + .unwrap(); + + crate::test_wait::wait_for_async( + || observer.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .await + .unwrap(); + // Leave room for a duplicate before checking there is none. + rt::sleep(Duration::from_millis(50)).await; + let exits = observer.request(GetExits).await.unwrap(); + assert_eq!(exits.len(), 1, "expected exactly one Exit, got {:?}", exits); + + let oh = observer.child_handle(); + oh.stop(); + oh.wait_exit_async().await; + }); + }, + ); } #[test] pub fn start_linked_links_atomically() { - // Parent + child via start_linked. When parent stops via panic, - // child (not trapping) should die too. - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - async fn handle(&mut self, _msg: PanicParent, _ctx: &Context) { - panic!("parent boom"); - } - } - - struct Child; - impl Actor for Child {} + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_links_atomically"), + move || { + // Parent + child via start_linked. When parent stops via panic, + // child (not trapping) should die too. + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + async fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } + } - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); - - // Parent panics โ€” child should die too - let _ = parent.send(PanicParent); - parent.wait_exit().await; - // Child should also die (link propagation) - let reason = child.wait_exit().await; - // Reason should be the parent's panic - assert!(matches!(reason, ExitReason::Panic(_))); - }); + struct Child; + impl Actor for Child {} + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); + + // Parent panics โ€” child should die too + let _ = parent.send(PanicParent); + parent.wait_exit().await; + // Child should also die (link propagation) + let reason = child.wait_exit().await; + // Reason should be the parent's panic + assert!(matches!(reason, ExitReason::Panic(_))); + }); + }, + ); } } @@ -2825,26 +4416,41 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn closed_monitor_table_rejects_late_registration() { - let rt = rt::Runtime::new().unwrap(); - let _entered = rt.enter(); - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.shared.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.shared.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - rt.block_on(observer.join()); - rt.block_on(target.join()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let rt = rt::Runtime::new().unwrap(); + let _entered = rt.enter(); + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.shared.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.shared.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + rt.block_on(observer.join()); + rt.block_on(target.join()); + }, + ); } fn check_cleanup(stop_owner: bool) { let rt = rt::Runtime::new().unwrap(); @@ -2900,11 +4506,16 @@ mod early_stop_regression { } #[test] fn panic_during_early_stop_is_reported() { - let rt = rt::Runtime::new().unwrap(); - let _enter = rt.enter(); - let actor = EarlyStop.start(); - assert!( - matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let rt = rt::Runtime::new().unwrap(); + let _enter = rt.enter(); + let actor = EarlyStop.start(); + assert!( + matches!(rt.block_on(actor.wait_exit()), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, ); } } @@ -2915,109 +4526,131 @@ mod turn_regressions { use std::sync::atomic::{AtomicUsize, Ordering}; #[test] fn idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield() { - let mut turn = MessageTurn { - limit: 16, - adaptive: false, - slice: Some(Duration::from_micros(50)), - completed: 0, - began: Some(std::time::Instant::now()), - }; - turn.parked(); - assert!(turn.began.is_none()); - let wake = std::time::Instant::now(); - turn.resumed(); - assert!(turn.began.unwrap() >= wake); - futures::executor::block_on(turn.after_message(true)); - assert!(turn.began.is_none()); - assert_eq!(turn.completed, 0); + crate::test_wait::bounded( + concat!( + module_path!(), + "::idle_time_is_excluded_and_an_empty_mailbox_needs_no_yield" + ), + move || { + let mut turn = MessageTurn { + limit: 16, + adaptive: false, + slice: Some(Duration::from_micros(50)), + completed: 0, + began: Some(std::time::Instant::now()), + }; + turn.parked(); + assert!(turn.began.is_none()); + let wake = std::time::Instant::now(); + turn.resumed(); + assert!(turn.began.unwrap() >= wake); + futures::executor::block_on(turn.after_message(true)); + assert!(turn.began.is_none()); + assert_eq!(turn.completed, 0); + }, + ); } #[test] fn count_only_turn_never_starts_a_clock_and_yields_at_its_count() { - rt::Runtime::new().unwrap().block_on(async { - let mut turn = MessageTurn { - limit: 4, - adaptive: false, - slice: None, - completed: 0, - began: None, - }; - turn.resumed(); - for _ in 0..3 { - let mut message = Box::pin(turn.after_message(false)); - assert!(futures::poll!(message.as_mut()).is_ready()); - } - assert!(turn.began.is_none()); - let mut fourth = Box::pin(turn.after_message(false)); - assert!(futures::poll!(fourth.as_mut()).is_pending()); - fourth.await; - assert_eq!(turn.completed, 0); - assert!(turn.began.is_none()); - turn.parked(); - turn.resumed(); - assert!(turn.began.is_none()); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::count_only_turn_never_starts_a_clock_and_yields_at_its_count" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let mut turn = MessageTurn { + limit: 4, + adaptive: false, + slice: None, + completed: 0, + began: None, + }; + turn.resumed(); + for _ in 0..3 { + let mut message = Box::pin(turn.after_message(false)); + assert!(futures::poll!(message.as_mut()).is_ready()); + } + assert!(turn.began.is_none()); + let mut fourth = Box::pin(turn.after_message(false)); + assert!(futures::poll!(fourth.as_mut()).is_pending()); + fourth.await; + assert_eq!(turn.completed, 0); + assert!(turn.began.is_none()); + turn.parked(); + turn.resumed(); + assert!(turn.began.is_none()); + }); + }, + ); } #[test] fn stop_during_batch_yield_does_not_execute_the_next_queued_message() { - struct Count(Arc); - impl Actor for Count {} - struct Inc; - impl Message for Inc { - type Result = (); - } - impl Handler for Count { - async fn handle(&mut self, _: Inc, _: &Context) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (_, completion_rx) = watch::channel(None); - let ctx = Context { - shared: Arc::new(ActorShared { - id: ActorId::next(), - sender: tx.clone(), - cancellation_token: token.clone(), - completion_rx, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - managed: None, - }), - }; - let count = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { - tx.send(Box::new(MessageEnvelope { - msg: Inc, - tx: None, - #[cfg(feature = "metrics")] - observation: None, - }) as Box + Send>) - .unwrap(); - } - let turn = MessageTurn { - limit: 1, - adaptive: false, - slice: Some(Duration::from_micros(50)), - completed: 0, - began: None, - }; - let mut job = Box::pin(run_actor( - Count(count.clone()), - ctx, - rx, - token.clone(), - Some(turn), - )); - assert!(futures::poll!(job.as_mut()).is_pending()); - assert_eq!(count.load(Ordering::SeqCst), 1); - token.cancel(); - assert_eq!(job.await, Ok(ExitReason::Normal)); - assert_eq!(count.load(Ordering::SeqCst), 1); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_during_batch_yield_does_not_execute_the_next_queued_message" + ), + move || { + struct Count(Arc); + impl Actor for Count {} + struct Inc; + impl Message for Inc { + type Result = (); + } + impl Handler for Count { + async fn handle( + &mut self, + _: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); + } + } + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (ctx, mut cleanup) = test_context(&tx, &token); + let count = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + tx.send(Box::new(MessageEnvelope { + msg: Some(Inc), + slot: SlotHandle::detached(), + lent: None, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }) + as Box + Send>) + .unwrap(); + } + let turn = MessageTurn { + limit: 1, + adaptive: false, + slice: Some(Duration::from_micros(50)), + completed: 0, + began: None, + }; + let mut job = Box::pin(run_actor( + Count(count.clone()), + ctx, + rx, + token.clone(), + &mut cleanup, + Some(turn), + )); + assert!(futures::poll!(job.as_mut()).is_pending()); + assert_eq!(count.load(Ordering::SeqCst), 1); + token.cancel(); + job.await; + assert_eq!(cleanup.die(), ExitReason::Normal); + assert_eq!(count.load(Ordering::SeqCst), 1); + }); + }, + ); } } @@ -3032,59 +4665,64 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, _: Increment, _: &Context) { + async fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } #[test] fn stop_while_idle_does_not_dispatch_a_queued_request_on_resume() { - rt::Runtime::new().unwrap().block_on(async { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (_, completion_rx) = watch::channel(None); - let ctx = Context { - shared: Arc::new(ActorShared { - id: ActorId::next(), - sender: tx.clone(), - cancellation_token: token.clone(), - completion_rx, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - managed: None, - }), - }; - let count = Arc::new(AtomicUsize::new(0)); - let mut actor = Box::pin(run_actor( - Counter(count.clone()), - ctx, - rx, - token.clone(), - #[cfg(feature = "backend-experiments")] - None, - )); - // Start and park the actual actor loop, without allowing another - // executor task to run it between admission and stop below. - assert!(futures::poll!(actor.as_mut()).is_pending()); - let (reply, response) = oneshot::channel(); - tx.send(Box::new(MessageEnvelope { - msg: Increment, - tx: Some(reply), - #[cfg(feature = "metrics")] - observation: None, - }) as Box + Send>) - .unwrap(); - token.cancel(); - assert_eq!(actor.await, Ok(ExitReason::Normal)); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "idle actor dispatched queued work after stop" - ); - assert!(response.await.is_err(), "queued request unexpectedly ran"); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_idle_does_not_dispatch_a_queued_request_on_resume" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (ctx, mut cleanup) = test_context(&tx, &token); + let count = Arc::new(AtomicUsize::new(0)); + let mut actor = Box::pin(run_actor( + Counter(count.clone()), + ctx, + rx, + token.clone(), + &mut cleanup, + #[cfg(feature = "backend-experiments")] + None, + )); + // Start and park the actual actor loop, without allowing another + // executor task to run it between admission and stop below. + assert!(futures::poll!(actor.as_mut()).is_pending()); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); + tx.send(Box::new(MessageEnvelope { + msg: Some(Increment), + slot, + lent: None, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }) as Box + Send>) + .unwrap(); + token.cancel(); + actor.await; + assert_eq!(cleanup.die(), ExitReason::Normal); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "idle actor dispatched queued work after stop" + ); + assert!(response.await.is_err(), "queued request unexpectedly ran"); + }); + }, + ); } } @@ -3115,136 +4753,206 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - async fn handle(&mut self, msg: Ask, _: &Context) -> u32 { - msg.0 + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); } } impl Handler for Echo { - async fn handle(&mut self, msg: StopAfter, ctx: &Context) { + async fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { let _ = msg.0.await; ctx.stop(); + reply.reply(()); } } impl Handler for Echo { async fn handle( &mut self, _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, ctx: &Context, - ) -> (Recipient, ActorId) { - (ctx.recipient::(), ctx.id()) + ) { + reply.reply((ctx.recipient::(), ctx.id())); } } impl Handler for Echo { - async fn handle(&mut self, _: Boom, _: &Context) { - panic!("boom"); + async fn handle( + &mut self, + _: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("boom"); + } } } #[test] fn context_recipient_reaches_its_actor() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join().await; - assert!(matches!( - recipient.send(Ask(1)), - Err(ActorError::ActorStopped) - )); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::context_recipient_reaches_its_actor"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).await.unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT).await, Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }); + }, + ); } #[test] fn requests_to_a_stopped_actor_fail_without_waiting() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join().await; - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT).await, - Err(ActorError::ActorStopped) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT).await, - Err(ActorError::ActorStopped) - )); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join().await; + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT).await, + Err(ActorError::NoProc) + )); + }); + }, + ); } #[test] fn admitted_requests_fail_when_the_actor_stops_before_replying() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); - let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); - assert!(futures::poll!(direct.as_mut()).is_pending()); - assert!(futures::poll!(erased.as_mut()).is_pending()); - release.send(()).unwrap(); - assert!(matches!(direct.await, Err(ActorError::ActorStopped))); - assert!(matches!(erased.await, Err(ActorError::ActorStopped))); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_actor_stops_before_replying" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + let mut direct = Box::pin(actor.request_with_timeout(Ask(1), WAIT)); + let mut erased = Box::pin(request(&*recipient, Ask(2), WAIT)); + assert!(futures::poll!(direct.as_mut()).is_pending()); + assert!(futures::poll!(erased.as_mut()).is_pending()); + release.send(()).unwrap(); + assert!(matches!( + direct.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + assert!(matches!( + erased.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + }); + }, + ); } #[test] fn erased_request_times_out() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = rt::oneshot::channel(); - actor.send(StopAfter(held)).unwrap(); - assert!(matches!( - request(&*recipient, Ask(1), Duration::from_millis(1)).await, - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::erased_request_times_out"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = rt::oneshot::channel(); + actor.send(StopAfter(held)).unwrap(); + assert!(matches!( + request(&*recipient, Ask(1), Duration::from_millis(1)).await, + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join().await; + }); + }, + ); } #[test] fn default_exit_received_keeps_a_trapping_actor_alive() { - struct Trapper; - impl Actor for Trapper { - async fn started(&mut self, ctx: &Context) { - ctx.trap_exit(true); - } - } - impl Handler for Trapper { - async fn handle(&mut self, msg: Ask, _: &Context) -> u32 { - msg.0 - } - } - rt::Runtime::new().unwrap().block_on(async { - let trapper = Trapper.start(); - assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); - assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + struct Trapper; + impl Actor for Trapper { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + } + impl Handler for Trapper { + async fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); + } + } + rt::Runtime::new().unwrap().block_on(async { + let trapper = Trapper.start(); + assert!(matches!(trapper.request(Ask(0)).await, Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit().await, ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)).await, Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit().await, ExitReason::Shutdown); + }); + }, + ); } #[test] fn debug_output_names_the_handle() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join().await; - }); + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join().await; + }); + }, + ); } } @@ -3269,7 +4977,14 @@ mod shared_state_release { struct Plain; impl Actor for Plain {} impl Handler for Plain { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } /// Keeps a clone of its own `Context` in its state. @@ -3282,57 +4997,75 @@ mod shared_state_release { #[test] fn freed_after_stop_and_last_handle_dropped() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Plain.start(); - let weak = actor.shared_weak(); - let extra = (actor.clone(), actor.context(), actor.child_handle()); - actor.context().stop(); - actor.join().await; - drop(extra); - drop(actor); - assert!(freed(&weak).await, "ActorShared outlived every handle"); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::freed_after_stop_and_last_handle_dropped"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Plain.start(); + let weak = actor.shared_weak(); + let extra = (actor.clone(), actor.context(), actor.child_handle()); + actor.context().stop(); + actor.join().await; + drop(extra); + drop(actor); + assert!(freed(&weak).await, "ActorShared outlived every handle"); + }); + }, + ); } #[test] fn freed_when_context_was_stored_in_actor_state() { - rt::Runtime::new().unwrap().block_on(async { - let actor = Holder(None).start(); - let weak = actor.shared_weak(); - actor.context().stop(); - actor.join().await; - drop(actor); - assert!(freed(&weak).await, "Context kept in actor state leaked"); - }); + crate::test_wait::bounded( + concat!( + module_path!(), + "::freed_when_context_was_stored_in_actor_state" + ), + move || { + rt::Runtime::new().unwrap().block_on(async { + let actor = Holder(None).start(); + let weak = actor.shared_weak(); + actor.context().stop(); + actor.join().await; + drop(actor); + assert!(freed(&weak).await, "Context kept in actor state leaked"); + }); + }, + ); } #[test] fn freed_with_links_and_monitors() { - rt::Runtime::new().unwrap().block_on(async { - let a = Plain.start(); - let b = Plain.start(); - let watcher = Plain.start(); - a.context().trap_exit(true); - a.context().link(&b.child_handle()); - let _monitor = watcher.context().monitor(&a.child_handle()); - watcher.context().monitor(&b.child_handle()); - let child = Plain.start_linked(&a.context()); - let weaks = [ - a.shared_weak(), - b.shared_weak(), - watcher.shared_weak(), - child.shared_weak(), - ]; - for actor in [&b, &a, &watcher] { - actor.context().stop(); - actor.join().await; - } - child.context().stop(); - child.join().await; - drop((a, b, watcher, child)); - for weak in &weaks { - assert!(freed(weak).await, "linked or monitored actor leaked"); - } - }); + crate::test_wait::bounded( + concat!(module_path!(), "::freed_with_links_and_monitors"), + move || { + rt::Runtime::new().unwrap().block_on(async { + let a = Plain.start(); + let b = Plain.start(); + let watcher = Plain.start(); + a.context().trap_exit(true); + a.context().link(&b.child_handle()); + let _monitor = watcher.context().monitor(&a.child_handle()); + watcher.context().monitor(&b.child_handle()); + let child = Plain.start_linked(&a.context()); + let weaks = [ + a.shared_weak(), + b.shared_weak(), + watcher.shared_weak(), + child.shared_weak(), + ]; + for actor in [&b, &a, &watcher] { + actor.context().stop(); + actor.join().await; + } + child.context().stop(); + child.join().await; + drop((a, b, watcher, child)); + for weak in &weaks { + assert!(freed(weak).await, "linked or monitored actor leaked"); + } + }); + }, + ); } } diff --git a/concurrency/src/tasks/jobs.rs b/concurrency/src/tasks/jobs.rs new file mode 100644 index 000000000..16f0b2da3 --- /dev/null +++ b/concurrency/src/tasks/jobs.rs @@ -0,0 +1,1115 @@ +//! `tasks` mode pipe-to-self jobs (docs/design/deferred-replies.md ยง9): +//! `ctx.pipe*` and `ctx.pipe_reply*`, their envelopes and `JobCtx`. +//! +//! A child module of `tasks::actor`, so it reaches the private envelope and +//! shared-state types. + +use super::{dispatched, Actor, Context, Dispatch, Envelope, Handler}; +use crate::message::Message; +use crate::reply::deadline::Expirable; +use crate::reply::jobs::{ + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, + PipeError, PipeReplyError, SendScheduled, +}; +use crate::reply::slot::{Refused, Slot}; +use crate::reply::{now_ticks, PendingReply, ReplyTo}; +use futures::future::{self, FutureExt as _}; +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::pin::pin; +use std::sync::{Arc, Mutex, Weak}; + +// --------------------------------------------------------------------------- +// Envelopes +// --------------------------------------------------------------------------- + +/// Returns one emit credit when the emitted message is dispatched, fenced or +/// dropped with the mailbox. +struct EmitCredit(Arc); +impl Drop for EmitCredit { + fn drop(&mut self) { + self.0.return_emit_credit(); + } +} + +/// A message emitted by a job; fenced at dispatch when the job is cancelled +/// or belongs to another instance. +struct EmitEnvelope { + msg: Option, + credit: Option, +} + +impl Envelope for EmitEnvelope +where + A: Actor + Handler, + M: Message, +{ + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let msg = dispatched(&mut self.msg); + let credit = dispatched(&mut self.credit); + let fenced = credit.0.fenced(ctx.id(), "emit"); + // The credit returns at dispatch (or here, when fenced). + drop(credit); + if fenced { + ctx.shared.sender.failures().discard(msg); + return (Box::pin(async {}), None); + } + ( + Box::pin(actor.handle(msg, PendingReply::detached(), ctx)), + None, + ) + } +} + +type Done = Box) -> M + Send>; + +/// A job's terminal message: `done(outcome)` runs in the owner's turn and the +/// produced message is dispatched in the same turn. +struct TerminalEnvelope { + job: Arc, + /// `done` and the outcome, taken at dispatch. + payload: Option<(Done, JobOutcome)>, +} + +impl Envelope for TerminalEnvelope +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + #[cfg(feature = "diagnostics")] + fn message_type(&self) -> &'static str { + std::any::type_name::() + } + fn handle<'a>(&'a mut self, actor: &'a mut A, ctx: &'a Context) -> Dispatch<'a> { + let Self { job, payload } = self; + let job: &'a Arc = job; + let (done, outcome) = dispatched(payload); + let kind = if matches!(outcome, Err(JobError::DeadlineExceeded)) { + "deadline" + } else { + "done" + }; + // Everything runs inside the returned future, so a panicking `done` + // is a handler panic caught by the run loop (`stopped()`, turn end + // and job cancellation still run). + let turn = Box::pin(async move { + if job.fenced(ctx.id(), kind) { + // Fenced: `done` and its captures (a linked token) are + // discarded in the turn, through the mailbox drop path. + ctx.shared.sender.failures().discard((done, outcome)); + return; + } + let msg = done(outcome); + actor.handle(msg, PendingReply::detached(), ctx).await + }); + (turn, None) + } +} + +// --------------------------------------------------------------------------- +// Job runs +// --------------------------------------------------------------------------- + +/// The parts shared by a job's completion path and its deadline. +struct Run { + state: Arc, + ctx: Context, + done: Mutex>>, + /// The job's deadline (the heap holds a `Weak`): owned until the run is + /// dropped, so it still fires while a result waits for scheduled emits. + deadline: std::sync::OnceLock>, + /// What execution held (its credit, or its place in a linked slot), + /// handed over when execution ends and released when the terminal + /// message is enqueued (or the run is dropped): a result waiting for + /// scheduled emits keeps it, so `MAX_OUTSTANDING` still bounds it. + hold: Mutex>>, +} + +fn end_of(outcome: &JobOutcome) -> &'static str { + match outcome { + Ok(_) => "ok", + Err(JobError::Panicked(_)) => "panicked", + Err(JobError::DeadlineExceeded) => "deadline", + } +} + +impl Run +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + /// Take the terminal notification and enqueue `(done, outcome)`. + fn notify(&self, outcome: JobOutcome) { + let (_, unsent) = self + .state + .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); + // A refused terminal message (`done` and the outcome) may drop on + // the deadline service thread: contained. + self.ctx.shared.sender.failures().discard(unsent); + } + + fn enqueue_terminal(&self, outcome: JobOutcome) -> Option + Send>> { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + // Execution ended (or is still running past its deadline, and then + // keeps its hold): return the credit before the terminal message + // can be dispatched, so `done`'s turn can start the next job at the + // bound. Only atomics run here (a credit or the slot's job count). + let hold = self.hold.lock().unwrap_or_else(|p| p.into_inner()).take(); + drop(hold); + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + payload: Some((done, outcome)), + }); + self.ctx.shared.sender.send(envelope).err().map(|e| e.0) + } + + /// The work's result: notify now, or, while delayed emits are still + /// scheduled, hold it for the last of them so the terminal message + /// stays last. `hold` is what execution held; the terminal message + /// releases it. + fn finish(self: &Arc, outcome: JobOutcome, hold: Box) { + *self.hold.lock().unwrap_or_else(|p| p.into_inner()) = Some(hold); + let end = end_of(&outcome); + let outcome = std::cell::Cell::new(Some(outcome)); + let unsent = self.state.finish( + end, + || self.enqueue_terminal(outcome.take()?), + || { + let run = self.clone(); + // Dropped unrun (a deadline or a cancel overtook it) possibly + // on the deadline service thread: contained. + let outcome = outcome.take().map(|outcome| { + crate::reply::jobs::Contained::new(outcome, self.ctx.shared.sender.failures()) + }); + Box::new(move || { + if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { + run.notify(outcome); + } + }) + }, + ); + self.ctx.shared.sender.failures().discard(unsent); + } +} + +impl Drop for Run { + /// The last reference may go on the deadline service thread (a deadline + /// or a held result firing): `done` and its captures drop contained. + fn drop(&mut self) { + let done = self + .done + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); + self.ctx.shared.sender.failures().discard(done); + } +} + +/// Holds a `pipe_reply` job's place in its slot's `JOBS` count. +struct LinkedJob(Arc>); +impl Drop for LinkedJob { + fn drop(&mut self) { + self.0.job_end(); + } +} + +/// What a job holds while it executes; released when execution ends. +enum Hold { + Credit(#[allow(dead_code)] JobCredit), + Linked(#[allow(dead_code)] LinkedJob), +} + +/// Native slot (`MAX_NATIVE_JOBS`) held by a blocking job. +struct NativeSlot { + ctx: Context, + /// Where the job will run, reserved with the slot (`None` once used). + runner: Option, +} +impl Drop for NativeSlot { + fn drop(&mut self) { + self.ctx.shared.jobs.release_native(); + } +} + +/// Where a blocking job's body runs, fixed when its native slot is reserved +/// so that launching it cannot fail (threads mode reserves its thread the +/// same way). +struct NativeRunner { + /// A managed actor's scope lease, taken when the job is admitted (never + /// by an unused permit, which must not keep a dead actor's scope owned) + /// and held until the body physically ends: the actor's cleanup is not + /// complete (no supervisor replacement, no complete system shutdown) + /// while its native work still runs. + lease: Option, + pool: RunnerPool, +} + +enum RunnerPool { + /// Tokio's blocking pool: the managed actor's group runtime, or the + /// current one. + #[cfg(not(spawned_sim))] + Tokio(spawned_rt::tasks::Handle), + /// Outside a Tokio runtime: an owned thread, spawned at reservation and + /// parked until the job starts (dropping the sender unused lets it exit). + #[cfg(not(spawned_sim))] + Thread(std::sync::mpsc::Sender>), + /// Under the simulator: `spawned_rt::tasks::spawn_blocking`, which runs + /// the body inline as one step of a simulated task (determinism D2), so + /// its clock reads, emits and result are part of the seeded schedule. + #[cfg(spawned_sim)] + Sim, +} + +impl NativeRunner { + /// Run `task`, then release the lease (after the body, and everything + /// it drops, ended). + fn run(self, task: impl FnOnce() + Send + 'static) { + let NativeRunner { lease, pool } = self; + let task = move || { + task(); + drop(lease); + }; + match pool { + #[cfg(not(spawned_sim))] + RunnerPool::Tokio(handle) => drop(handle.spawn_blocking(task)), + // The thread is parked on `recv` until this send (or the + // sender's drop); it cannot have exited. + #[cfg(not(spawned_sim))] + RunnerPool::Thread(start) => { + let _ = start.send(Box::new(task)); + } + #[cfg(spawned_sim)] + RunnerPool::Sim => drop(spawned_rt::tasks::spawn_blocking(task)), + } + } +} + +/// Marks the job ended in the owner's job set when execution ends. +struct Ended(Context, Arc); +impl Drop for Ended { + fn drop(&mut self) { + self.0.shared.jobs.ended(&self.1); + } +} + +fn caught(result: std::thread::Result) -> JobOutcome { + result.map_err(|panic| { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobError::Panicked(message) + }) +} + +/// The job's handle on its owner while it runs. +pub struct JobCtx { + state: Arc, + ctx: Context, +} + +impl std::fmt::Debug for JobCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobCtx") + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobCtx { + /// Whether the job was cancelled (handle, key replacement, owner exit, + /// or opt-in caller give-up). Cancellation suppresses later results. + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + + /// Resolves when the job is cancelled. + pub async fn cancelled(&self) { + self.state.cancellation().cancelled().await; + } + + /// Send `msg` to the owner as one of this job's messages, fenced like the + /// terminal one and never after it. Waits while `emit_credits` messages + /// are in flight; fails with [`JobGone`] once the job is cancelled or its + /// terminal notification was taken. + pub async fn emit(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + std::future::poll_fn(|cx| self.state.poll_acquire(cx)).await?; + self.enqueue(msg) + } + + /// [`emit`](Self::emit) delivered at `at` instead of now (OTP's + /// `erlang:send_after` to the owner, fenced with the job). Takes an emit + /// credit now (waiting while all are in flight) and keeps it until the + /// message is dispatched. At `at` the message is enqueued like an + /// `emit`: dropped if the job was cancelled or replaced, or its deadline + /// passed. The job's terminal message waits for its scheduled emits, so + /// it stays last even when the work finishes first. Fails with + /// [`JobGone`] at the call once the job is cancelled or its notification + /// was taken. + /// + /// `at` is an instant of the monotonic deadline clock. One already + /// reached (or in the past) is an immediate emit: the message is + /// enqueued at the call, ahead of later emits, so there is no need to + /// branch to [`emit`](Self::emit). An `at` past `JobOptions::deadline` + /// is accepted; the deadline fences it (the emit is dropped at `at`). + pub async fn emit_at(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + std::future::poll_fn(|cx| self.state.poll_acquire(cx)).await?; + self.schedule(at, msg) + } + + /// Blocking [`emit_at`](Self::emit_at), for `pipe_blocking*` jobs. + pub fn emit_at_blocking(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.schedule(at, msg) + } + + fn schedule(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + let ctx = self.ctx.clone(); + let send: SendScheduled = Box::new(move || { + let envelope: Box + Send> = Box::new(EmitEnvelope { + msg: Some(msg), + credit: Some(credit), + }); + ctx.shared + .sender + .send(envelope) + .map_err(|refused| Box::new(refused.0) as Box) + }); + JobState::schedule_emit( + &self.state, + spawned_rt::time::to_ticks(at), + send, + self.ctx.shared.sender.failures(), + |at, item| self.ctx.shared.replies.track(at, item), + ) + } + + /// Blocking [`emit`](Self::emit), for `pipe_blocking*` jobs. + pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.enqueue(msg) + } + + fn enqueue(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + self.state + .enqueue_emit(|| { + let envelope: Box + Send> = Box::new(EmitEnvelope { + msg: Some(msg), + credit: Some(credit), + }); + self.ctx.shared.sender.send(envelope) + }) + .map_err(|(gone, unsent)| { + drop(unsent); + gone + }) + } +} + +// --------------------------------------------------------------------------- +// Context API +// --------------------------------------------------------------------------- + +/// A job admission reserved ahead of building the work: +/// `ctx.reserve_job(opts)` takes the credit (and checks the owner is not +/// stopping) without consuming anything, so a caller that is refused still +/// holds what it would have moved into `work` and `done`. Spawning from a +/// permit cannot fail. Dropping an unused permit returns its credit. +#[must_use = "an unused permit only holds a credit; spawn the job with it or drop it"] +pub struct JobPermit { + ctx: Context, + opts: JobOptions, + credit: JobCredit, +} + +/// [`JobPermit`] for a blocking job: it also holds a native slot +/// (`MAX_NATIVE_JOBS`). From `ctx.reserve_blocking_job(opts)`. +#[must_use = "an unused permit only holds a credit and a native slot; spawn the job with it or drop it"] +pub struct BlockingJobPermit { + ctx: Context, + opts: JobOptions, + credit: JobCredit, + native: NativeSlot, +} + +impl std::fmt::Debug for JobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobPermit").finish_non_exhaustive() + } +} + +impl std::fmt::Debug for BlockingJobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockingJobPermit").finish_non_exhaustive() + } +} + +impl JobPermit { + /// [`Context::pipe`] with this permit's options and credit; cannot + /// fail. If the owner began exiting since the reservation, nothing runs: + /// `work` and `done` are dropped and the handle reports the job + /// cancelled (as owner exit cancels every job). + pub fn pipe(self, work: F, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + F: FnOnce(JobCtx) -> W, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let JobPermit { ctx, opts, credit } = self; + let Some(state) = ctx.start_job(&opts) else { + return ctx.refused_job(&opts, (work, done)); + }; + let job = JobCtx { + state: state.clone(), + ctx: ctx.clone(), + }; + let work = work(job); + // Admitted after `work` was built (user code): an owner that exited + // meanwhile refuses the job, which then never runs. + if !ctx.admit_job(&state, &opts) { + return ctx.refuse(state, (work, done, credit)); + } + ctx.launch_async::( + opts, + state, + Hold::Credit(credit), + work, + Box::new(done), + None, + ) + } +} + +impl BlockingJobPermit { + /// [`Context::pipe_blocking`] with this permit's options, credit and + /// native slot; cannot fail. If the owner began exiting since the + /// reservation, nothing runs (see [`JobPermit::pipe`]). + pub fn pipe_blocking(self, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let BlockingJobPermit { + ctx, + opts, + credit, + mut native, + } = self; + let Some(state) = ctx.start_job(&opts) else { + return ctx.refused_job(&opts, (work, done, native)); + }; + if !ctx.admit_native(&state, &opts, &mut native) { + return ctx.refuse(state, (work, done, native, credit)); + } + ctx.launch_blocking::( + opts, + state, + Hold::Credit(credit), + native, + work, + Box::new(done), + ) + } +} + +impl Context { + /// The job's state, or `None` once the owner is exiting. + fn start_job(&self, opts: &JobOptions) -> Option> { + (!self.shared.replies.is_exiting()).then(|| JobState::new(self.id(), opts.emit_credits)) + } + + /// A permit used after the owner began exiting: nothing runs, the parts + /// drop here (outside locks), and the handle reports cancelled. + fn refused_job

(&self, opts: &JobOptions, parts: P) -> JobHandle { + self.refuse(JobState::new(self.id(), opts.emit_credits), parts) + } + + /// `state`'s admission was refused: as [`refused_job`](Self::refused_job). + fn refuse

(&self, state: Arc, parts: P) -> JobHandle { + drop(parts); + state.never_started(); + JobHandle::new(state) + } + + /// Admit `state` into the owner's job set (the model's `Start`), after + /// every reversible reservation and before anything that cannot be + /// undone: refused once the owner's exit closed admission, in one step + /// with that close, so an admitted job is always cancelled (and joined) + /// by the owner's exit. + fn admit_job(&self, state: &Arc, opts: &JobOptions) -> bool { + self.shared + .jobs + .admit(state, opts.key.clone(), opts.deadline.is_some()) + } + + fn reserve_credit(&self) -> Result { + if self.shared.replies.is_exiting() { + return Err(PipeError::OwnerStopping); + } + JobCredit::reserve(&self.shared.replies).ok_or(PipeError::Overloaded) + } + + /// Reserve a native slot (`MAX_NATIVE_JOBS`) and where the job will run: + /// the blocking pool, or an owned thread spawned now (`SpawnFailed` + /// here, never at launch). A failure releases the slot. A managed + /// actor's scope lease is taken later, at admission + /// ([`admit_native`](Self::admit_native)). + fn reserve_native(&self) -> Result, PipeError> { + if !self.shared.jobs.reserve_native() { + return Err(PipeError::NativeBudget); + } + let mut slot = NativeSlot { + ctx: self.clone(), + runner: None, + }; + #[cfg(not(spawned_sim))] + let handle = match &self.shared.managed { + Some(scope) => Some(scope.handle.clone()), + None => spawned_rt::tasks::Handle::try_current().ok(), + }; + #[cfg(spawned_sim)] + let pool = RunnerPool::Sim; + #[cfg(not(spawned_sim))] + let pool = match handle { + Some(handle) => RunnerPool::Tokio(handle), + None => { + let (start, started) = std::sync::mpsc::channel::>(); + std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + if let Ok(task) = started.recv() { + task(); + } + }) + .map_err(PipeError::SpawnFailed)?; + RunnerPool::Thread(start) + } + }; + slot.runner = Some(NativeRunner { lease: None, pool }); + Ok(slot) + } + + /// Admit a blocking job: a managed actor's scope lease (refused once + /// its scope is closing, as the owner exits), then + /// [`admit_job`](Self::admit_job). The lease goes to the native runner + /// and lives until the body ends; refused, it is released here. + fn admit_native( + &self, + state: &Arc, + opts: &JobOptions, + native: &mut NativeSlot, + ) -> bool { + let lease = match &self.shared.managed { + Some(scope) => match scope.job_lease() { + Ok(lease) => Some(lease), + Err(_) => return false, + }, + None => None, + }; + if !self.admit_job(state, opts) { + return false; + } + if let Some(runner) = &mut native.runner { + runner.lease = lease; + } + true + } + + /// Reserve an unlinked job's admission (the owner is running, one + /// `MAX_OUTSTANDING` credit) before building `work` and `done`: a + /// refusal leaves the caller everything it would have moved into them, + /// and spawning from the permit cannot fail. + /// + /// ```ignore + /// let Ok(permit) = ctx.reserve_job(JobOptions::new()) else { + /// return self.report(id, Verdict::Overloaded); // `id` was not moved + /// }; + /// permit.pipe(|_| fetch(object), move |res| Fetched { id, res }); + /// ``` + pub fn reserve_job(&self, opts: JobOptions) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); + Ok(JobPermit { + ctx: self.clone(), + credit: self.reserve_credit()?, + opts, + }) + } + + /// [`reserve_job`](Self::reserve_job) for a blocking job: also takes a + /// native slot (`MAX_NATIVE_JOBS`, `NativeBudget` at the bound). + pub fn reserve_blocking_job( + &self, + opts: JobOptions, + ) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); + let credit = self.reserve_credit()?; + let native = self.reserve_native()?; + Ok(BlockingJobPermit { + ctx: self.clone(), + opts, + credit, + native, + }) + } + + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) + where + A: Handler, + M: Message, + T: Send + 'static, + { + let Some(at) = opts.deadline.map(spawned_rt::time::to_ticks) else { + return; + }; + let weak = Arc::downgrade(run); + let deadline = Arc::new(JobDeadline { + at, + state: run.state.clone(), + notify: Mutex::new(Some(Box::new(move || { + if let Some(run) = weak.upgrade() { + run.notify(Err(JobError::DeadlineExceeded)); + } + }))), + }); + // The run owns it; the heap only refers to it. + let _ = run.deadline.set(deadline.clone()); + if at <= now_ticks() { + deadline.expire(at); + } else { + let item: Weak = Arc::downgrade(&deadline) as Weak; + self.shared.replies.track(at, item); + } + } + + fn launch_async( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + work: W, + done: Done, + caller_gone: Option>>, + ) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: Future + Send + 'static, + { + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), + }); + self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let handle = JobHandle::new(state.clone()); + self.spawn_framework(async move { + let _ended = ended; + let cancelled = pin!(state.cancellation().cancelled()); + let gone = pin!(async { + match &caller_gone { + Some(slot) => { + std::future::poll_fn(|cx| slot.poll_closed(cx.waker(), &now_ticks)).await + } + None => std::future::pending().await, + } + }); + let work = pin!(AssertUnwindSafe(work).catch_unwind()); + let stop = future::select(cancelled, gone); + match future::select(work, stop).await { + // Execution ended: its credit (and linked-job count) goes to + // the run, which returns it when the terminal message is + // enqueued. + future::Either::Left((result, _)) => run.finish(caught(result), Box::new(hold)), + // Cancelled (or the opt-in caller give-up): the work future is + // dropped here, and with it any downstream receiver. + future::Either::Right(_) => { + state.cancel(); + state.ended_cancelled(); + } + } + }); + handle + } + + fn launch_blocking( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + mut native: NativeSlot, + work: W, + done: Done, + ) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + { + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), + }); + self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + let runner = native + .runner + .take() + .expect("a native slot is reserved with its runner"); + runner.run(move || { + let _ended = ended; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(job))); + // Execution ended: release the native slot now; the run returns + // the credit when the terminal message is enqueued. + drop(native); + run.finish(caught(result), Box::new(hold)); + }); + JobHandle::new(state) + } + + /// Run async work as a job owned by this actor instance and deliver + /// `done(outcome)` as one message in a later turn (`handle_info`), as + /// `Task.async` + `handle_info` would. `work` receives the job's + /// [`JobCtx`] (to check cancellation and `emit` fenced messages, bounded + /// by `emit_credits`; the terminal message is always last) and returns + /// the future to run; `|_| fut` when it needs neither. A panic in the + /// future arrives as `Err(JobError::Panicked)`. + /// + /// `work` itself is a factory that runs **at once, in this turn, on the + /// calling thread**: only the future it returns is the job. Code before + /// its `async` block runs in the handler, so a panic there is a handler + /// panic (the actor's panic boundary, not `JobError::Panicked`), and if + /// the owner's exit closes admission meanwhile the job is refused after + /// the factory ran (its future is dropped unpolled). Put fallible or slow + /// preparation inside the `async` block. (`pipe_blocking` runs its + /// closure on the job's thread, under the job's panic boundary.) + /// + /// Holds one `MAX_OUTSTANDING` credit + /// while it executes. Rejects at once at a bound; never waits. On `Err`, + /// `work` and `done` are dropped: use [`reserve_job`](Self::reserve_job) + /// to keep them. + /// + /// ```ignore + /// ctx.pipe(JobOptions::new().key("fetch"), |_| fetch(url), Fetched)?; + /// ``` + pub fn pipe( + &self, + opts: JobOptions, + work: F, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + F: FnOnce(JobCtx) -> W, + W: Future + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + Ok(self.reserve_job(opts)?.pipe(work, done)) + } + + /// Run blocking `work` on Tokio's blocking pool, under the actor's + /// `MAX_NATIVE_JOBS` budget (cancellation stays cooperative: the job + /// checks its [`JobCtx`]). `work` gets the [`JobCtx`] to check + /// cancellation and `emit_blocking` fenced messages; a panic arrives as + /// `JobError::Panicked`. On `Err`, `work` and `done` are dropped: use + /// [`reserve_blocking_job`](Self::reserve_blocking_job) to keep them. + pub fn pipe_blocking( + &self, + opts: JobOptions, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + Ok(self.reserve_blocking_job(opts)?.pipe_blocking(work, done)) + } + + /// Delegate a deferred reply: run async work as a job linked to + /// `token`'s request and deliver `done(outcome, token)` in a later turn. + /// `work` receives the [`JobCtx`] as in [`pipe`](Self::pipe). The + /// obligation and credit stay in this actor: the token and its first + /// linked job share one credit. With + /// [`JobOptions::cancel_on_caller_gone`] the job is cancelled when the + /// caller leaves or the request's deadline passes (off by default, as + /// in OTP). On every error the token comes back. + pub fn pipe_reply( + &self, + opts: JobOptions, + token: ReplyTo, + work: F, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + F: FnOnce(JobCtx) -> W, + W: Future + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link_reply(&opts, token)?; + let watch = opts.cancel_on_caller_gone.then(|| slot.clone()); + #[cfg(feature = "lifecycle-trace")] + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); + let hold = Hold::Linked(LinkedJob(slot)); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + // User code; a panic here drops the hold, which detaches the + // linked job from the slot. + let work = work(job); + // Reply traces: requests the work makes are this request's + // downstream calls. + #[cfg(feature = "lifecycle-trace")] + let work = { + let mut work = Box::pin(work); + std::future::poll_fn(move |cx| { + crate::lifecycle_trace::in_link(Some(link), || work.as_mut().poll(cx)) + }) + }; + if !self.admit_job(&state, &opts) { + // Dropping the hold detaches the linked job from the slot. + drop((hold, watch, work)); + return Err(PipeReplyError::Failed { + token, + error: PipeError::OwnerStopping, + }); + } + #[cfg(feature = "lifecycle-trace")] + state.linked(link.1); + Ok(self.launch_async(opts, state, hold, work, linked_done(token, done), watch)) + } + + /// Blocking [`pipe_reply`](Self::pipe_reply) on Tokio's blocking pool, + /// under the `MAX_NATIVE_JOBS` budget. + pub fn pipe_reply_blocking( + &self, + opts: JobOptions, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link_reply(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot.clone())); + let mut native = match self.reserve_native() { + Ok(native) => native, + // Dropping the hold detaches the linked job from the slot. + Err(error) => { + drop(hold); + return Err(PipeReplyError::Failed { token, error }); + } + }; + if !self.admit_native(&state, &opts, &mut native) { + drop((hold, native)); + return Err(PipeReplyError::Failed { + token, + error: PipeError::OwnerStopping, + }); + } + if opts.cancel_on_caller_gone { + crate::reply::jobs::cancel_when_closed(&slot, &state); + } + #[cfg(feature = "lifecycle-trace")] + let work = { + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); + state.linked(link.1); + move |job| crate::lifecycle_trace::in_link(Some(link), || work(job)) + }; + Ok(self.launch_blocking(opts, state, hold, native, work, linked_done(token, done))) + } + + /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the + /// attach CAS on the token's slot (which reserves a credit only when a + /// linked job already runs). + #[allow(clippy::type_complexity)] + fn link_reply( + &self, + opts: &JobOptions, + token: ReplyTo, + ) -> Result<(Arc, Arc>, ReplyTo), PipeReplyError> { + if token.owner() != self.id() { + return Err(PipeReplyError::WrongOwner(token)); + } + let Some(state) = self.start_job(opts) else { + return Err(PipeReplyError::Failed { + token, + error: PipeError::OwnerStopping, + }); + }; + let slot = token.slot_arc().clone(); + let attached = slot.attach(&now_ticks); + // The slot refused the job (reply traces: `PipeReply` handing the + // token back). + #[cfg(feature = "lifecycle-trace")] + if attached.is_err() { + crate::lifecycle_trace::reply( + self.id().raw(), + "PipeReply", + Arc::as_ptr(&slot) as u64, + "refused", + ); + } + match attached { + Ok(()) => Ok((state, slot, token)), + Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), + Err(Refused::Overloaded) => Err(PipeReplyError::Failed { + token, + error: PipeError::Overloaded, + }), + } + } +} + +fn linked_done(token: ReplyTo, done: D) -> Done +where + R: Send + 'static, + T: Send + 'static, + M: Message, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, +{ + Box::new(move |outcome| done(outcome, token)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tasks::ActorStart; + + struct Owner; + impl Actor for Owner {} + struct Note; + impl Message for Note { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Note, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review P1 (Codex 13): an emit refused after its credit was taken + /// (the job was cancelled in between) returns that credit once, through + /// its guard. It used to be returned by the refusal and again by the + /// guard, so the job ended with more credits than it was given (and + /// `u16::MAX` credits overflowed). + #[tokio::test] + async fn a_refused_emit_returns_its_credit_once() { + crate::test_wait::within( + concat!(module_path!(), "::a_refused_emit_returns_its_credit_once"), + async move { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join().await; + }, + ) + .await; + } + + /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no + /// caller, so setting it there is a bug, caught in debug builds. + #[cfg(debug_assertions)] + #[tokio::test] + async fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { + crate::test_wait::within( + concat!( + module_path!(), + "::cancel_on_caller_gone_on_a_plain_job_is_a_bug" + ), + async move { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join().await; + }, + ) + .await; + } +} diff --git a/concurrency/src/tasks/mod.rs b/concurrency/src/tasks/mod.rs index f3c9698f5..074d221d6 100644 --- a/concurrency/src/tasks/mod.rs +++ b/concurrency/src/tasks/mod.rs @@ -7,10 +7,19 @@ mod stream_tests; #[cfg(test)] mod timer_tests; +pub use crate::reply::jobs::{ + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, +}; +pub use crate::reply::{ + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyFuture, ReplyReceiver, ReplyTo, +}; pub use crate::response::Response; +pub use crate::Instant; pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Backend, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; +pub use actor::{BlockingJobPermit, JobCtx, JobPermit, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/tasks/stream_tests.rs b/concurrency/src/tasks/stream_tests.rs index f7f183767..98b411dd8 100644 --- a/concurrency/src/tasks/stream_tests.rs +++ b/concurrency/src/tasks/stream_tests.rs @@ -43,194 +43,243 @@ impl Summatory { impl Actor for Summatory {} impl Handler for Summatory { - async fn handle(&mut self, msg: StreamMsg, ctx: &Context) { + async fn handle( + &mut self, + msg: StreamMsg, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { match msg { StreamMsg::Add(val) => self.count += val, StreamMsg::Error => ctx.stop(), } + reply.reply(()); } } impl Handler for Summatory { - async fn handle(&mut self, _msg: StopSum, ctx: &Context) { + async fn handle( + &mut self, + _msg: StopSum, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop(); + reply.reply(()); } } impl Handler for Summatory { - async fn handle(&mut self, _msg: GetValue, _ctx: &Context) -> u16 { - self.count + async fn handle( + &mut self, + _msg: GetValue, + reply: crate::reply::PendingReply<'_, u16>, + _ctx: &Context, + ) { + reply.reply(self.count); } } #[test] pub fn test_sum_numbers_from_stream() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = stream::iter(vec![1u16, 2, 3, 4, 5].into_iter().map(Ok::)); - - let ctx = Context::from_ref(&summatory); - spawn_listener( - ctx, - stream.filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_stream"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = stream::iter(vec![1u16, 2, 3, 4, 5].into_iter().map(Ok::)); + + let ctx = summatory.context(); + spawn_listener( + ctx, + stream.filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_sum_numbers_from_channel() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); - - spawned_rt::tasks::spawn(async move { - for i in 1..=5 { - tx.send(Ok(i)).unwrap(); - } - }); - - let ctx = Context::from_ref(&summatory); - spawn_listener( - ctx, - ReceiverStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_channel"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); + + spawned_rt::tasks::spawn(async move { + for i in 1..=5 { + tx.send(Ok(i)).unwrap(); + } + }); + + let ctx = summatory.context(); + spawn_listener( + ctx, + ReceiverStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_sum_numbers_from_broadcast_channel() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = tokio::sync::broadcast::channel::(5); - - spawned_rt::tasks::spawn(async move { - for i in 1u16..=5 { - tx.send(i).unwrap(); - } - }); - - let ctx = Context::from_ref(&summatory); - spawn_listener( - ctx, - BroadcastStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_sum_numbers_from_broadcast_channel"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = tokio::sync::broadcast::channel::(5); + + spawned_rt::tasks::spawn(async move { + for i in 1u16..=5 { + tx.send(i).unwrap(); + } + }); + + let ctx = summatory.context(); + spawn_listener( + ctx, + BroadcastStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } #[test] pub fn test_stream_cancellation() { - const MESSAGE_INTERVAL: u64 = 250; - const READ_TIME: u64 = 850; - const STOP_TIME: u64 = 1100; - - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); - - spawned_rt::tasks::spawn(async move { - for i in 1..=5 { - tx.send(Ok(i)).unwrap(); - rt::sleep(Duration::from_millis(MESSAGE_INTERVAL)).await; - } - }); - - let ctx = Context::from_ref(&summatory); - let listener_handle = spawn_listener( - ctx.clone(), - ReceiverStream::new(rx) - .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), - ); - - let _ = send_after(Duration::from_millis(STOP_TIME), ctx, StopSum); - - rt::sleep(Duration::from_millis(READ_TIME)).await; - let val = summatory.request(GetValue).await.unwrap(); - - assert!((1..=15).contains(&val)); - - assert!(listener_handle.await.is_ok()); - - summatory.join().await; - assert!(summatory.request(GetValue).await.is_err()); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_stream_cancellation"), + move || { + const MESSAGE_INTERVAL: u64 = 250; + const READ_TIME: u64 = 850; + const STOP_TIME: u64 = 1100; + + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let (tx, rx) = spawned_rt::tasks::mpsc::channel::>(); + + spawned_rt::tasks::spawn(async move { + for i in 1..=5 { + tx.send(Ok(i)).unwrap(); + rt::sleep(Duration::from_millis(MESSAGE_INTERVAL)).await; + } + }); + + let ctx = summatory.context(); + let listener_handle = spawn_listener( + ctx.clone(), + ReceiverStream::new(rx) + .filter_map(|result| async move { result.ok().map(StreamMsg::Add) }), + ); + + let _ = send_after(Duration::from_millis(STOP_TIME), ctx, StopSum); + + rt::sleep(Duration::from_millis(READ_TIME)).await; + let val = summatory.request(GetValue).await.unwrap(); + + assert!((1..=15).contains(&val)); + + assert!(listener_handle.await.is_ok()); + + summatory.join().await; + assert!(summatory.request(GetValue).await.is_err()); + }) + }, + ); } #[test] pub fn test_halting_on_stream_error() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); - let msg_stream = stream.filter_map(|value| async move { - match value { - Ok(number) => Some(StreamMsg::Add(number)), - Err(_) => Some(StreamMsg::Error), - } - }); - - let ctx = Context::from_ref(&summatory); - spawn_listener(ctx, msg_stream); - - // The error message stops the actor. - summatory.join().await; - - let result = summatory.request(GetValue).await; - assert!(result.is_err()); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_halting_on_stream_error"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = + tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); + let msg_stream = stream.filter_map(|value| async move { + match value { + Ok(number) => Some(StreamMsg::Add(number)), + Err(_) => Some(StreamMsg::Error), + } + }); + + let ctx = summatory.context(); + spawn_listener(ctx, msg_stream); + + // The error message stops the actor. + summatory.join().await; + + let result = summatory.request(GetValue).await; + assert!(result.is_err()); + }) + }, + ); } #[test] pub fn test_skipping_on_stream_error() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let summatory = Summatory::new(0).start(); - let stream = tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); - let msg_stream = stream.filter_map(|value| async move { - match value { - Ok(number) => Some(StreamMsg::Add(number)), - Err(_) => None, - } - }); - - let ctx = Context::from_ref(&summatory); - spawn_listener(ctx, msg_stream); - - let val = wait_for_async( - || summatory.request(GetValue), - |val| matches!(val, Ok(n) if *n >= 15), - ) - .await - .unwrap(); - assert_eq!(val, 15); - }) + crate::test_wait::bounded( + concat!(module_path!(), "::test_skipping_on_stream_error"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let summatory = Summatory::new(0).start(); + let stream = + tokio_stream::iter(vec![Ok(1u16), Ok(2), Ok(3), Err(()), Ok(4), Ok(5)]); + let msg_stream = stream.filter_map(|value| async move { + match value { + Ok(number) => Some(StreamMsg::Add(number)), + Err(_) => None, + } + }); + + let ctx = summatory.context(); + spawn_listener(ctx, msg_stream); + + let val = wait_for_async( + || summatory.request(GetValue), + |val| matches!(val, Ok(n) if *n >= 15), + ) + .await + .unwrap(); + assert_eq!(val, 15); + }) + }, + ); } diff --git a/concurrency/src/tasks/timer_tests.rs b/concurrency/src/tasks/timer_tests.rs index 40902cabe..894affde4 100644 --- a/concurrency/src/tasks/timer_tests.rs +++ b/concurrency/src/tasks/timer_tests.rs @@ -46,51 +46,73 @@ impl Actor for Repeater { } impl Handler for Repeater { - async fn handle(&mut self, _msg: Inc, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.count += 1; + reply.reply(()); } } impl Handler for Repeater { - async fn handle(&mut self, _msg: StopTimer, _ctx: &Context) { + async fn handle( + &mut self, + _msg: StopTimer, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } + reply.reply(()); } } impl Handler for Repeater { - async fn handle(&mut self, _msg: GetRepCount, _ctx: &Context) -> i32 { - self.count + async fn handle( + &mut self, + _msg: GetRepCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + reply.reply(self.count); } } #[test] pub fn test_send_interval_and_cancellation() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Repeater::new(0).start(); - - // The interval keeps firing. - let count = wait_for_async( - || repeater.request(GetRepCount), - |count| matches!(count, Ok(n) if *n >= 3), - ) - .await - .unwrap(); - assert!(count >= 3, "interval stopped firing at {count}"); - - // StopTimer is handled before the next request. A tick that raced the - // cancellation may still land, but no more than one. - repeater.send(StopTimer).unwrap(); - let stopped_at = repeater.request(GetRepCount).await.unwrap(); - rt::sleep(Duration::from_millis(500)).await; - let after = repeater.request(GetRepCount).await.unwrap(); - assert!( - after <= stopped_at + 1, - "interval kept firing after cancellation: {stopped_at} -> {after}" - ); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_interval_and_cancellation"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Repeater::new(0).start(); + + // The interval keeps firing. + let count = wait_for_async( + || repeater.request(GetRepCount), + |count| matches!(count, Ok(n) if *n >= 3), + ) + .await + .unwrap(); + assert!(count >= 3, "interval stopped firing at {count}"); + + // StopTimer is handled before the next request. A tick that raced the + // cancellation may still land, but no more than one. + repeater.send(StopTimer).unwrap(); + let stopped_at = repeater.request(GetRepCount).await.unwrap(); + rt::sleep(Duration::from_millis(500)).await; + let after = repeater.request(GetRepCount).await.unwrap(); + assert!( + after <= stopped_at + 1, + "interval kept firing after cancellation: {stopped_at} -> {after}" + ); + }); + }, + ); } // --- Delayed (send_after test) --- @@ -122,77 +144,106 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - async fn handle(&mut self, _msg: Inc, _ctx: &Context) { + async fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.count += 1; + reply.reply(()); } } impl Handler for Delayed { - async fn handle(&mut self, _msg: GetDelCount, _ctx: &Context) -> i32 { - self.count + async fn handle( + &mut self, + _msg: GetDelCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + reply.reply(self.count); } } impl Handler for Delayed { - async fn handle(&mut self, _msg: StopDelayed, ctx: &Context) -> i32 { - ctx.stop(); - self.count + async fn handle( + &mut self, + _msg: StopDelayed, + reply: crate::reply::PendingReply<'_, i32>, + ctx: &Context, + ) { + let __reply_value: i32 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); } } #[test] pub fn test_send_after_and_cancellation() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_and_cancellation"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Delayed::new(0).start(); - let ctx = Context::from_ref(&repeater); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for_async( - || repeater.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .await - .unwrap(); - assert_eq!(1, count); + let count = wait_for_async( + || repeater.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .await + .unwrap(); + assert_eq!(1, count); - let ctx = Context::from_ref(&repeater); - let timer = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = repeater.context(); + let timer = send_after(Duration::from_millis(100), ctx, Inc); - timer.cancellation_token.cancel(); + timer.cancellation_token.cancel(); - rt::sleep(Duration::from_millis(200)).await; + rt::sleep(Duration::from_millis(200)).await; - let count2 = repeater.request(GetDelCount).await.unwrap(); - assert_eq!(1, count2); - }); + let count2 = repeater.request(GetDelCount).await.unwrap(); + assert_eq!(1, count2); + }); + }, + ); } #[test] pub fn test_send_after_gen_server_teardown() { - let runtime = rt::Runtime::new().unwrap(); - runtime.block_on(async move { - let repeater = Delayed::new(0).start(); - - let ctx = Context::from_ref(&repeater); - let _ = send_after(Duration::from_millis(100), ctx, Inc); - - let count = wait_for_async( - || repeater.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .await - .unwrap(); - assert_eq!(1, count); - - let ctx = Context::from_ref(&repeater); - let _ = send_after(Duration::from_millis(100), ctx, Inc); - - let count2 = repeater.request(StopDelayed).await.unwrap(); - - rt::sleep(Duration::from_millis(200)).await; - - assert_eq!(1, count2); - }); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_gen_server_teardown"), + move || { + let runtime = rt::Runtime::new().unwrap(); + runtime.block_on(async move { + let repeater = Delayed::new(0).start(); + + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); + + let count = wait_for_async( + || repeater.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .await + .unwrap(); + assert_eq!(1, count); + + let ctx = repeater.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); + + let count2 = repeater.request(StopDelayed).await.unwrap(); + + rt::sleep(Duration::from_millis(200)).await; + + assert_eq!(1, count2); + }); + }, + ); } diff --git a/concurrency/src/test_wait.rs b/concurrency/src/test_wait.rs index 9138ec6bc..6c875d608 100644 --- a/concurrency/src/test_wait.rs +++ b/concurrency/src/test_wait.rs @@ -40,3 +40,44 @@ where spawned_rt::tasks::sleep(POLL).await; } } + +/// Bound for a wait a bug can turn into a hang (`tests/support/deadline.rs` +/// uses the same): far above any healthy wait, which are all handshakes. +pub(crate) const HANG: Duration = Duration::from_secs(120); + +/// Run a test's body (its runtime and that runtime's teardown included) on +/// its own thread, named after the test. A body still running after [`HANG`] +/// fails the test by name instead of hanging the job; a panic in the body +/// (a failed assertion, or a `#[should_panic]` test's expected panic) is +/// re-raised unchanged. +pub(crate) fn bounded( + name: &str, + body: impl FnOnce() -> T + Send + 'static, +) -> T { + let (done, finished) = std::sync::mpsc::channel(); + let thread = std::thread::Builder::new() + .name(name.to_string()) + .spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(body)); + let _ = done.send(result); + }) + .expect("spawn the test body"); + match finished.recv_timeout(HANG) { + Ok(Ok(value)) => { + let _ = thread.join(); + value + } + Ok(Err(panic)) => std::panic::resume_unwind(panic), + Err(_) => panic!("timed out after {HANG:?}: {name} did not finish"), + } +} + +/// Async counterpart of [`bounded`] for `#[tokio::test]` bodies: the body's +/// awaits are bounded by [`HANG`] (the runtime's own teardown is not). +#[cfg(feature = "async")] +pub(crate) async fn within(name: &str, body: F) -> F::Output { + match spawned_rt::tasks::timeout(HANG, body).await { + Ok(output) => output, + Err(_) => panic!("timed out after {HANG:?}: {name} did not finish"), + } +} diff --git a/concurrency/src/threads/actor.rs b/concurrency/src/threads/actor.rs index 0a07cf9f4..394767025 100644 --- a/concurrency/src/threads/actor.rs +++ b/concurrency/src/threads/actor.rs @@ -1,25 +1,36 @@ use crate::mailbox::threads as mpsc; -use spawned_rt::threads::{self as rt, oneshot, oneshot::RecvTimeoutError, CancellationToken}; +use spawned_rt::threads::{self as rt, CancellationToken}; +#[cfg(test)] +use std::{collections::HashMap, sync::Mutex}; use std::{ - collections::HashMap, fmt::Debug, panic::{catch_unwind, AssertUnwindSafe}, - sync::{Arc, Mutex}, + sync::Arc, time::Duration, }; -use crate::child_handle::{ActorId, ChildHandle}; -use crate::error::{panic_message, ActorError, ExitReason}; +use crate::child_handle::{ActorId, ChildHandle, Completion}; +use crate::error::{panic_message, ActorError, ExitReason, NoProc}; use crate::link::{ - self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkTable, - LinkedExitReason, SendExitFn, TrapExitFlag, + self, new_link_table, new_linked_exit_reason, new_trap_exit_flag, Exit, LinkOutcome, LinkTable, + LinkedExitReason, SendExitFn, SendHeldFn, SignalTarget, TrapExitFlag, }; use crate::message::Message; use crate::monitor::{Down, MonitorCleanup, MonitorRef, MonitorTable}; +use crate::reply::{registry::Registry, reply_slot, SlotHandle}; +use crate::reply::{Deadline, PendingReply, ReplyReceiver, DEFAULT_MAX_OUTSTANDING}; use std::sync::atomic::Ordering; pub use crate::response::DEFAULT_REQUEST_TIMEOUT; +/// Cleanup deadline for actors without a supervisor: the bound on joining an +/// exiting actor's cancelled jobs. +pub const DEFAULT_STOP_DEADLINE: Duration = Duration::from_secs(5); + +#[path = "jobs.rs"] +mod jobs; +pub use jobs::{BlockingJobPermit, JobCtx}; + // --------------------------------------------------------------------------- // Actor trait // --------------------------------------------------------------------------- @@ -31,6 +42,21 @@ pub use crate::response::DEFAULT_REQUEST_TIMEOUT; /// /// Actors must be `Send + Sized + 'static` so they can be moved to a spawned thread. pub trait Actor: Send + Sized + 'static { + /// Deferred-reply credits per instance (live `ReplyTo` tokens plus running + /// linked jobs). At the bound `defer()` answers the caller `Overloaded`. + /// Default 1024; capped at 65535. + const MAX_OUTSTANDING: usize = DEFAULT_MAX_OUTSTANDING; + + /// Native threads this instance's jobs may run at once. `None`: + /// `min(4, available_parallelism)`. + /// + /// The default is small: an actor that runs many blocking jobs at once + /// usually needs more, and under load the only symptom is + /// `PipeError::NativeBudget` refusals (the first one per instance is + /// logged as a `native_budget_exhausted` warning). Set it with + /// `#[actor(max_native_jobs = N)]`. + const MAX_NATIVE_JOBS: Option = None; + fn started(&mut self, _ctx: &Context) {} fn stopped(&mut self, _ctx: &Context) {} @@ -46,8 +72,13 @@ pub trait Actor: Send + Sized + 'static { /// Per-message handler trait. Implement once for each message type the actor handles. /// /// Unlike the `tasks` version, handlers are synchronous โ€” no `async`/`.await`. +/// +/// `handle` is the envelope dispatch for both `send` and `request`: answer +/// through `reply` ([`PendingReply::reply`], or [`PendingReply::defer`] to +/// answer later). A request whose `PendingReply` is dropped unanswered gets +/// `ReplyDropped`; a sent message has a detached `PendingReply`. pub trait Handler: Actor { - fn handle(&mut self, msg: M, ctx: &Context) -> M::Result; + fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, ctx: &Context); } // --------------------------------------------------------------------------- @@ -55,6 +86,11 @@ pub trait Handler: Actor { // --------------------------------------------------------------------------- trait Envelope: Send { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "other" + } #[cfg(feature = "metrics")] fn observation(&mut self) -> Option { None @@ -62,14 +98,61 @@ trait Envelope: Send { #[cfg(feature = "metrics")] fn reject(self: Box) {} + /// The owner reached its death point with this envelope undispatched: + /// answer its request now (the final reason), before any payload of the + /// mailbox is destroyed. No user code runs. + fn discharge(&mut self, _replies: Option<&Registry>) {} + fn handle(self: Box, actor: &mut A, ctx: &Context); } +/// Answer every request queued in a dead owner's mailbox (and `extra`, an +/// envelope received after the stop), destroying nothing: the items come +/// back, to be destroyed once the death point's wake-ups ran. Admission is +/// closed first (waiting for in-flight sends), so nothing is queued after. +fn answer_mailbox( + rx: &mut mpsc::Receiver>, + extra: Option + Send>>, + replies: Option<&Registry>, +) -> Vec> { + let mut queued = rx.drain(); + queued.extend(extra.map(MailboxItem::Message)); + for item in &mut queued { + if let MailboxItem::Message(envelope) = item { + envelope.discharge(replies); + } + } + queued +} + +/// A never-polled actor's startup payload (its thread was never started). +type StartupPayload = ( + A, + Context, + mpsc::Receiver>, + CancellationToken, +); +fn discharge_startup(payload: &mut StartupPayload) -> Vec> { + let replies = payload.1.replies.clone(); + erase(answer_mailbox(&mut payload.2, None, Some(&replies))) +} +fn erase(items: Vec) -> Vec> { + items + .into_iter() + .map(|item| Box::new(item) as Box) + .collect() +} + struct MessageEnvelope { + // Before the message: a plain drop also answers the request first. + slot: SlotHandle, msg: M, - tx: Option>, #[cfg(feature = "metrics")] observation: Option, + /// The recorded actor that sent it (0: outside code or a request); its + /// mailbox item kind is then the model's `msg`. + #[cfg(feature = "lifecycle-trace")] + from: u64, } impl Envelope for MessageEnvelope @@ -77,6 +160,15 @@ where A: Actor + Handler, M: Message, { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + if self.from != 0 { + "msg" + } else { + crate::lifecycle_trace::life_kind::() + } + } #[cfg(feature = "metrics")] fn observation(&mut self) -> Option { self.observation.take() @@ -87,49 +179,148 @@ where observation.reject(); } } + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } fn handle(self: Box, actor: &mut A, ctx: &Context) { - let result = actor.handle(self.msg, ctx); - if let Some(tx) = self.tx { - let delivered = tx.send(result).is_ok(); - #[cfg(feature = "metrics")] - crate::instrumentation::reply(delivered); - #[cfg(not(feature = "metrics"))] - let _ = delivered; + let mut this = *self; + let reply = PendingReply::new(this.slot.take(), &ctx.replies); + actor.handle(this.msg, reply, ctx); + } +} + +impl> Envelope for DownEnvelope { + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "down" + } + #[cfg(feature = "metrics")] + fn observation(&mut self) -> Option { + self.observation.take() + } + #[cfg(feature = "metrics")] + fn reject(mut self: Box) { + if let Some(observation) = self.observation.take() { + observation.reject(); + } + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + fail_point!("monitor.down_before_dispatch"); + if crate::monitor::fence::take_queued(&ctx.monitors, &self.down.monitor_ref) { + let reply = PendingReply::new(None, &ctx.replies); + actor.handle(self.down, reply, ctx); } } } +/// Admit a message. Past the death point (the registry is sealed, or the +/// mailbox is gone) it reports `NoProc`; before it, including while the +/// actor exits, it is queued, and a request is answered at the death point +/// if it is not handled. fn enqueue( sender: &mpsc::Sender>, _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, msg: M, - tx: Option>, + slot: SlotHandle, ) -> Result<(), ActorError> where A: Actor + Handler, M: Message, { + let from = crate::lifecycle_trace::life_sender::(replies.owner().raw(), slot.is_detached()); let envelope: Box + Send> = Box::new(MessageEnvelope { msg, - tx, + slot, + #[cfg(feature = "metrics")] + observation: Some(_mailbox.enqueued()), + #[cfg(feature = "lifecycle-trace")] + from, + }); + admit::(sender, replies, envelope, from) +} + +/// A monitor's `Down`, enqueued by the framework (the waiter at the target's +/// publication, or `monitor` past the target's death point). Its dispatch +/// runs `Handler` only while the monitor's table entry is still +/// queued: a `demonitor` that returned first flushed it (OTP +/// `demonitor(Ref, [flush])`). A `Down` user code sends is an ordinary +/// message. +struct DownEnvelope { + down: Down, + #[cfg(feature = "metrics")] + observation: Option, +} + +fn enqueue_down>( + sender: &mpsc::Sender>, + _mailbox: &crate::instrumentation::Mailbox, + replies: &Registry, + down: Down, +) -> Result<(), ActorError> { + let envelope: Box + Send> = Box::new(DownEnvelope { + down, #[cfg(feature = "metrics")] observation: Some(_mailbox.enqueued()), }); - match sender.send(MailboxItem::Message(envelope)) { - Ok(()) => { + admit::(sender, replies, envelope, 0) +} + +/// Admission of a built envelope (`M` names its kind for the trace). +fn admit( + sender: &mpsc::Sender>, + replies: &Registry, + envelope: Box + Send>, + _from: u64, +) -> Result<(), ActorError> { + // Admission and its trace event are one step of the lifecycle trace. + let owner = replies.owner().raw(); + crate::lifecycle_trace::life_section(owner, || { + if replies.refuses_admission() { #[cfg(feature = "metrics")] - crate::instrumentation::accepted(); - Ok(()) + envelope.reject(); + crate::lifecycle_trace::life_send::(owner, false, _from); + return Err(ActorError::NoProc); } - Err(_error) => { - #[cfg(feature = "metrics")] - { - if let MailboxItem::Message(envelope) = _error.0 { - envelope.reject(); + match sender.send(MailboxItem::Message(envelope)) { + Ok(()) => { + #[cfg(feature = "metrics")] + crate::instrumentation::accepted(); + crate::lifecycle_trace::life_send::(owner, true, _from); + Ok(()) + } + Err(_error) => { + #[cfg(feature = "metrics")] + { + if let MailboxItem::Message(envelope) = _error.0 { + envelope.reject(); + } } + crate::lifecycle_trace::life_send::(owner, false, _from); + Err(ActorError::NoProc) } - Err(ActorError::ActorStopped) } + }) +} + +/// A closure run in one of the actor's turns (`spawned_test::threads::in_actor`). +type InTurn = Box) -> R + Send>; +struct InTurnEnvelope { + slot: SlotHandle, + f: InTurn, +} + +impl Envelope for InTurnEnvelope { + fn discharge(&mut self, replies: Option<&Registry>) { + self.slot.dispose(replies); + drop(std::mem::replace(&mut self.slot, SlotHandle::detached())); + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let InTurnEnvelope { f, mut slot } = *self; + let reply = PendingReply::new(slot.take(), &ctx.replies); + let value = f(actor, ctx); + let _ = reply.reply(value); } } @@ -140,11 +331,101 @@ struct ExitEnvelope { } impl Envelope for ExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "exit" + } fn handle(self: Box, actor: &mut A, ctx: &Context) { actor.exit_received(self.exit, ctx); } } +/// A parent exit (E3): when dispatched, exit begins with the parent's reason +/// and `stopped()` runs. `exit_received` never sees it. +struct ParentExitEnvelope { + exit: Exit, +} + +impl Envelope for ParentExitEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "pexit" + } + fn handle(self: Box, _actor: &mut A, ctx: &Context) { + ctx.parent_exit(self.exit.reason); + } +} + +/// An exit signal that arrived while `started()` ran, in its mailbox place +/// (phase 0). If the actor trapped exits when `started()` returned it is an +/// `Exit` (or parent exit); otherwise it was resolved then and is inert. +struct HeldEnvelope { + exit: Exit, + parent: bool, +} + +impl Envelope for HeldEnvelope { + /// The model's mailbox item kind (`Lifecycle.tla`), for the trace. + #[cfg(feature = "lifecycle-trace")] + fn trace_kind(&self) -> &'static str { + "held" + } + fn handle(self: Box, actor: &mut A, ctx: &Context) { + if !link::lock_request(&ctx.linked_reason).held_trapped() { + return; + } + if self.parent { + ctx.parent_exit(self.exit.reason); + } else { + actor.exit_received(self.exit, ctx); + } + } +} + +/// Build the signal target of a new actor instance. +fn signal_target( + id: ActorId, + parent: Option, + sender: &mpsc::Sender>, + trap: TrapExitFlag, + request: LinkedExitReason, + token: CancellationToken, +) -> Arc { + let exit_tx = sender.clone(); + let parent_tx = sender.clone(); + let held_tx = sender.clone(); + let send_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); + exit_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + let send_parent_exit: SendExitFn = Arc::new(move |exit: Exit| { + let envelope: Box + Send> = Box::new(ParentExitEnvelope { exit }); + parent_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + let send_held: SendHeldFn = Arc::new(move |exit: Exit, parent: bool| { + let envelope: Box + Send> = Box::new(HeldEnvelope { exit, parent }); + held_tx + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc) + }); + Arc::new(SignalTarget { + id, + parent, + trap, + request, + wake: Arc::new(move || token.cancel()), + send_exit, + send_parent_exit, + send_held, + }) +} + // A private wakeup is distinct from user envelopes and never invokes a handler. enum MailboxItem { Message(Box + Send>), @@ -177,7 +458,11 @@ pub struct Context { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + signal: Arc, mailbox: crate::instrumentation::Mailbox, + replies: Arc, + jobs: Arc, + cleaned: Arc, } impl Clone for Context { @@ -191,7 +476,11 @@ impl Clone for Context { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), + jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } } @@ -203,9 +492,8 @@ impl Debug for Context { } impl Context { - /// Create a `Context` from an `ActorRef`. Useful for setting up timers - /// or stream listeners from outside the actor. - pub fn from_ref(actor_ref: &ActorRef) -> Self { + /// The `Context` of `actor_ref`'s actor (public as `ActorRef::context`). + pub(crate) fn from_ref(actor_ref: &ActorRef) -> Self { Self { id: actor_ref.id, sender: actor_ref.sender.clone(), @@ -215,7 +503,11 @@ impl Context { trap_exit: actor_ref.trap_exit.clone(), links: actor_ref.links.clone(), linked_reason: actor_ref.linked_reason.clone(), + signal: actor_ref.signal.clone(), mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), } } @@ -227,46 +519,113 @@ impl Context { self.mailbox.depth() } + /// Deferred-reply credits in use by this instance: live deferred + /// tokens plus running linked jobs (bounded by `Actor::MAX_OUTSTANDING`). + /// Experimental in 0.8 (for tests and diagnostics; the `metrics` + /// feature exports the same count): it may change before 1.0. + pub fn deferred_outstanding(&self) -> usize { + self.replies.outstanding() + } + /// The actor's unique identity. pub fn id(&self) -> ActorId { self.id } - /// Signal the actor to stop. The current handler will finish, then - /// `stopped()` is called and the actor exits. + /// Self-exit: the current handler finishes, then `stopped()` is called + /// and the actor exits `Normal`. pub fn stop(&self) { self.stop_with_reason(ExitReason::Normal); } - /// Request a cooperative exit with an explicit reason, without unwinding. - /// The current handler finishes and `stopped()` runs. The first committed - /// stop/link reason wins; a cleanup panic can still make the exit abnormal. - /// `Shutdown` is non-abnormal; `Kill` requests an abnormal exit and can - /// trigger transient supervision or propagate through links. It does not - /// forcibly interrupt running Rust code. + /// Self-exit with an explicit reason (OTP `{stop, Reason, State}`), without + /// unwinding. The current handler finishes and `stopped()` runs. The first + /// requested self-exit or stop wins; until the death point a kill + /// replaces it with `Killed`, an untrapped exit signal with its reason, and + /// a panic (in the handler or `stopped()`) with `Panic`. `Kill` stays + /// `Kill` and propagates over links as `Kill` (trappable). pub fn stop_with_reason(&self, reason: ExitReason) { - link::stop_with_reason(&self.linked_reason, reason, || { + link::stop_with_reason(self.id, &self.linked_reason, reason, || { self.cancellation_token.cancel() }); } + /// A parent exit being dispatched: normal termination with its reason + /// (part of the model's `Dispatch`, so not recorded as a stop). + fn parent_exit(&self, reason: ExitReason) { + link::request_stop(&self.linked_reason, reason, || { + self.cancellation_token.cancel() + }); + } + + /// Send an exit signal from this actor to `target` (OTP + /// `exit_signal(Target, Reason)`); see [`ChildHandle::exit`] for the + /// delivery rules. From the target's `start_linked` parent the signal is a + /// parent exit (a trapping target ends normally, through `stopped()`). + /// Works until this actor's death point, including from `stopped()`; + /// inert after it. + pub fn exit(&self, target: &ChildHandle, reason: ExitReason) { + // Held until delivered: the death point waits for it, so a signal + // sent before death precedes the death notices (and none is sent + // after it). + let Some(_sending) = self.replies.signal_permit() else { + return; + }; + if link::lock_request(&self.linked_reason).is_dead() { + return; + } + fail_point!("actor.exit_before_deliver"); + target.exit_from(self.id, reason, false); + } + /// Send a fire-and-forget message to this actor. pub fn send(&self, msg: M) -> Result<(), ActorError> where A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, None) + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.sender, &self.mailbox, msg, Some(tx))?; + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) + } + + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -290,13 +649,10 @@ impl Context { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(duration) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -323,7 +679,11 @@ impl Context { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), + jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } @@ -333,7 +693,8 @@ impl Context { /// [`Context::demonitor`]. When the monitored actor stops, a [`Down`] /// message is delivered to this actor's mailbox via `Handler`. /// - /// If the target is already dead, a `Down` message is delivered immediately. + /// If the target is past its death point, exactly one + /// `Down { reason: NoProc }` is delivered at once (OTP `noproc`). /// /// Multiple independent monitors are allowed on the same target โ€” each /// call returns a distinct `MonitorRef`. @@ -355,18 +716,53 @@ impl Context { let monitor_ref = MonitorRef::next(); let active = spawned_rt::sync::CancellationToken::new(); - { - let mut monitors = self.monitors.lock().unwrap_or_else(|p| p.into_inner()); - // The token check makes registration inert once shutdown starts. - // The closed table below serializes registration with exit cleanup, - // even on paths that unwind before cancelling the owner token. - if self.cancellation_token.is_cancelled() { - return monitor_ref; - } - let Some(monitors) = monitors.as_mut() else { - return monitor_ref; + let observer = self.id.raw(); + let registered = crate::lifecycle_trace::life_section(observer, || { + let event = |x| { + crate::lifecycle_trace::LifeEvent::new("Monitor", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(x) }; - monitors.insert(monitor_ref, active.clone()); + crate::monitor::fence::register( + &self.monitors, + monitor_ref, + active.clone(), + // The token check makes registration inert once shutdown + // starts. The closed table serializes registration with exit + // cleanup, even on paths that unwind before cancelling the + // owner token. + || self.cancellation_token.is_cancelled(), + // Linearised against the target's death point (E4): past it + // the only notice is NoProc, delivered under the observer's + // lock. + || target.is_past_death(), + || { + crate::lifecycle_trace::life_emit(observer, || event("late")); + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x("NoProc") + .k("down") + .o("late") + }); + let _ = enqueue_down( + &self.sender, + &self.mailbox, + &self.replies, + Down { + monitor_ref, + actor: target.id(), + reason: ExitReason::NoProc, + }, + ); + }, + || crate::lifecycle_trace::life_emit(observer, || event("active")), + ) == crate::monitor::fence::Registration::Active + }); + if !registered { + return monitor_ref; } let target = target.clone(); @@ -388,21 +784,31 @@ impl Context { } }); let Some(reason) = reason else { return }; - // Remove the entry from the monitor table so it doesn't accumulate - // stale entries over the actor's lifetime. Done before delivery - // since `demonitor` is now a no-op for this monitor anyway. - let deliver = monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)) - .is_some(); - if deliver { - let _ = actor_ref.send(Down { - monitor_ref, - reason, + // Receiver-side fence (E6): the entry is removed and the Down + // enqueued in one critical section, so once `demonitor` returns + // no Down of this monitor is enqueued. + crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::deliver(&monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Deliver", observer) + .s(target.id().raw()) + .m(monitor_ref.raw()) + .x(crate::lifecycle_trace::life_reason(&reason)) + .k("down") + .o("death") + }); + let _ = enqueue_down( + &actor_ref.sender, + &actor_ref.mailbox, + &actor_ref.replies, + Down { + monitor_ref, + actor: target.id(), + reason, + }, + ); }); - } + }); }); monitor_ref @@ -410,94 +816,120 @@ impl Context { /// Cancel a previously-set monitor. /// - /// The monitor waiter is released promptly. - /// If the target hasn't yet died, no `Down` message will be delivered. - /// If a `Down` message has already been delivered (or queued), this is - /// a best-effort cancellation โ€” the message may still arrive. + /// A fence with OTP's `flush` (`demonitor(Ref, [flush])`): once it + /// returns, no `Down` of this monitor is enqueued or handled; one already + /// in the mailbox is skipped. Demonitoring an unknown, delivered or + /// already cancelled monitor does nothing. pub fn demonitor(&self, monitor_ref: MonitorRef) { - let active = self - .monitors - .lock() - .unwrap_or_else(|p| p.into_inner()) - .as_mut() - .and_then(|entries| entries.remove(&monitor_ref)); + let observer = self.id.raw(); + let active = crate::lifecycle_trace::life_section(observer, || { + crate::monitor::fence::demonitor(&self.monitors, &monitor_ref, || { + crate::lifecycle_trace::life_emit(observer, || { + crate::lifecycle_trace::LifeEvent::new("Demonitor", observer) + .m(monitor_ref.raw()) + }); + }) + }); // Waking the waiter must happen after releasing the table lock. if let Some(active) = active { active.cancel(); } } - /// Set up a bidirectional link with another actor. + /// Set up a bidirectional link with another actor (OTP `link/1`). /// - /// When either side dies abnormally, the other receives an exit signal. - /// By default the receiver is terminated by the signal. Call - /// [`trap_exit(true)`] to convert signals into `Exit` messages delivered - /// via [`Actor::exit_received`] instead. + /// When either side's exit is published, the other receives a link + /// notice with the final reason: a non-trapping peer ignores `Normal` and + /// otherwise exits with that reason without running `stopped()`; a + /// trapping peer gets an `Exit` via [`Actor::exit_received`]. /// - /// If the target is already dead with an abnormal reason, the exit - /// signal is delivered immediately without retaining a link. Repeating an - /// existing live link is a no-op; each attempt to link a closed peer reports - /// that peer's exit for the new attempt. + /// If the target is past its death point, this actor receives a `NoProc` + /// notice: an `Exit { reason: NoProc }` when trapping, otherwise it exits + /// `NoProc`. Use [`try_link`](Self::try_link) to get `Err(NoProc)` + /// instead. Inert once this actor's exit began (in `stopped()`, from a + /// helper after the stop, or past its death point): no record and no + /// notice, as `monitor`. An existing link, or a pending notice from that peer, makes + /// this a no-op. /// - /// [`trap_exit(true)`]: Self::trap_exit /// [`Actor::exit_received`]: Actor::exit_received pub fn link(&self, target: &ChildHandle) { - let own_signal = link::make_signal( - self.trap_exit.clone(), - self.own_cancel_fn(), - self.own_send_exit_fn(), - self.linked_reason.clone(), - ); - let peer_signal = link::make_signal( - target.trap_exit_flag().clone(), - target.cancel_fn().clone(), - target.send_exit_fn().clone(), - target.linked_reason().clone(), - ); link::register_link( - self.id, + &self.signal, &self.links, - own_signal, - target.id(), + target.signal_target(), target.links(), - peer_signal, + false, ); } - /// Remove a previously-set bidirectional link. + /// Link as [`link`](Self::link), but report a target past its death + /// point as `Err(NoProc)` and keep no record. + pub fn try_link(&self, target: &ChildHandle) -> Result<(), NoProc> { + match link::register_link( + &self.signal, + &self.links, + target.signal_target(), + target.links(), + true, + ) { + LinkOutcome::NoProc => Err(NoProc), + LinkOutcome::Linked | LinkOutcome::Unchanged => Ok(()), + } + } + + /// Remove a link. A fence (OTP `unlink/1`): once it returns, the link has + /// no further effect on this actor; an `Exit` already queued stays. pub fn unlink(&self, target: &ChildHandle) { link::unregister_link(self.id, &self.links, target.id(), target.links()); } - /// Control how exit signals from linked actors are handled. + /// Whether this actor's exit has begun or a signal that will end it is + /// already pending, for a trapping actor that wants to refuse late work. /// - /// - `false` (default): an abnormal exit signal from a linked actor cancels - /// this actor. - /// - `true`: the signal is converted to an `Exit` message and delivered to - /// [`Actor::exit_received`]. + /// `true` once: + /// - exit has begun: `ctx.stop()`, an administrative stop + /// ([`ChildHandle::stop`](crate::ChildHandle::stop)), a kill, an untrapped exit + /// signal or link notice, a panic; it stays `true` in `stopped()`; + /// - a parent exit is queued for this trapping actor (E3: the + /// supervisor's or system's shutdown, or a signal from its + /// `start_linked` parent). The parent exit is dispatched after the + /// messages queued before it, and handlers of those messages see + /// `true`. /// - /// Direct Kill requests cannot be trapped; propagated Kill notices can. - pub fn trap_exit(&self, enabled: bool) { - self.trap_exit.store(enabled, Ordering::Release); - } - - /// Build a type-erased cancel closure for this actor. - fn own_cancel_fn(&self) -> Arc { - let token = self.cancellation_token.clone(); - let reason = self.linked_reason.clone(); - Arc::new(move || link::normal_stop(&reason, || token.cancel())) + /// A trapped [`Exit`](crate::Exit) from a linked peer or a non-parent + /// `exit` is not a pending exit: `exit_received` decides. Signals that + /// arrive while `started()` runs count once `started()` returns, when + /// they are resolved. Without trapping, a pending signal has already + /// begun the exit, so this is also `true`. + /// + /// Advisory: a signal can arrive right after it returns `false`. It + /// takes the actor's exit-request lock; nothing is added per message. + pub fn exit_pending(&self) -> bool { + link::lock_request(&self.linked_reason).exit_pending() } - /// Build a type-erased `SendExitFn` that enqueues an `ExitEnvelope` - /// onto this actor's mailbox. - fn own_send_exit_fn(&self) -> SendExitFn { - let sender = self.sender.clone(); - Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) - }) + /// Control how exit signals (link notices and `exit` signals) are + /// handled; read when each signal arrives (OTP `trap_exit`). + /// + /// - `false` (default): `Normal` is ignored; any other reason ends this + /// actor after its current handler, without `stopped()`. + /// - `true`: the signal is queued as an `Exit` for + /// [`Actor::exit_received`]. + /// + /// Only a kill is untrappable. A `start_linked` parent's exit ends a + /// trapping actor normally, through `stopped()`. Signals that arrive while + /// `started()` runs are resolved when it returns. + pub fn trap_exit(&self, enabled: bool) { + let id = self.id.raw(); + crate::lifecycle_trace::life_section(id, || { + // Recorded only when it changes (the model's SetTrap). + if self.trap_exit.swap(enabled, Ordering::AcqRel) != enabled { + crate::lifecycle_trace::life_emit(id, || { + crate::lifecycle_trace::LifeEvent::new("Trap", id) + .x(crate::lifecycle_trace::flag(enabled)) + }); + } + }); } pub(crate) fn cancellation_token(&self) -> CancellationToken { @@ -505,6 +937,9 @@ impl Context { } } +impl crate::__private::Sealed for Context {} +impl crate::__private::Sealed for ActorRef {} + // Bridge: Context implements Receiver for any M that A handles impl Receiver for Context where @@ -515,8 +950,20 @@ where Context::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - Context::request_raw(self, msg) + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + Context::send_request_with(self, msg, deadline.ticks(), false) } } @@ -527,10 +974,28 @@ where /// Object-safe trait for sending a single message type to an actor. /// /// Implemented automatically by `ActorRef` and `Context` for any -/// message type that `A` handles. -pub trait Receiver: Send + Sync { +/// message type that `A` handles. Sealed: it cannot be implemented outside +/// spawned (a method can then be added without a breaking change). +pub trait Receiver: Send + Sync + crate::__private::Sealed { fn send(&self, msg: M) -> Result<(), ActorError>; - fn request_raw(&self, msg: M) -> Result, ActorError>; + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline); its + /// deadline is fixed now (the slot carries it). On a [`Recipient`], + /// pass a `Deadline` (`Deadline::after(d)`, or `d.into()`). + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError>; + /// `send_request` for the `request` helper, which arms its own timer: + /// the deadline does not go to the owner's heap. + #[doc(hidden)] + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + self.send_request(msg, deadline) + } } /// Type-erased reference for sending a single message type. @@ -543,13 +1008,9 @@ pub fn request( timeout: Duration, ) -> Result { let observation = crate::instrumentation::RequestObservation::new(); - let result = match recipient.request_raw(msg) { + let result = match recipient.send_request_timed(msg, Deadline::after(timeout)) { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(timeout) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -573,7 +1034,11 @@ pub struct ActorRef { trap_exit: TrapExitFlag, links: LinkTable, linked_reason: LinkedExitReason, + signal: Arc, mailbox: crate::instrumentation::Mailbox, + replies: Arc, + jobs: Arc, + cleaned: Arc, } impl Debug for ActorRef { @@ -593,7 +1058,11 @@ impl Clone for ActorRef { trap_exit: self.trap_exit.clone(), links: self.links.clone(), linked_reason: self.linked_reason.clone(), + signal: self.signal.clone(), mailbox: self.mailbox.clone(), + replies: self.replies.clone(), + jobs: self.jobs.clone(), + cleaned: self.cleaned.clone(), } } } @@ -612,17 +1081,48 @@ impl ActorRef { A: Handler, M: Message, { - enqueue(&self.sender, &self.mailbox, msg, None) + enqueue( + &self.sender, + &self.mailbox, + &self.replies, + msg, + SlotHandle::detached(), + ) + } + + /// Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get + /// its [`ReplyReceiver`]. The deadline is fixed now: a `Duration` from + /// now, an `Instant`, or [`Deadline::Never`] for an unbounded wait + /// ([`DEFAULT_REQUEST_TIMEOUT`] is what [`request`](Self::request) + /// uses). The handler sees it through `PendingReply::deadline`. + pub fn send_request( + &self, + msg: M, + deadline: impl Into, + ) -> Result, ActorError> + where + A: Handler, + M: Message, + { + let deadline = deadline.into(); + self.send_request_with(msg, deadline.ticks(), true) } - /// Send a request and get a raw oneshot receiver for the reply. - pub fn request_raw(&self, msg: M) -> Result, ActorError> + /// `send_request`; `raw` puts a deadline in the owner's heap now (the + /// receiver may be awaited directly). The `request` paths arm their own + /// timer and pass `false`. + pub(crate) fn send_request_with( + &self, + msg: M, + deadline: u64, + raw: bool, + ) -> Result, ActorError> where A: Handler, M: Message, { - let (tx, rx) = oneshot::channel(); - enqueue(&self.sender, &self.mailbox, msg, Some(tx))?; + let (slot, rx) = reply_slot(deadline, Some(&self.replies), raw); + enqueue(&self.sender, &self.mailbox, &self.replies, msg, slot)?; Ok(rx) } @@ -635,6 +1135,29 @@ impl ActorRef { self.request_with_timeout(msg, DEFAULT_REQUEST_TIMEOUT) } + /// Run `f` in one of the actor's turns, with its state and `Context`, + /// and receive its result. For tests: `spawned_test::threads::in_actor` (through + /// `__private`). + pub(crate) fn in_turn(&self, f: F) -> Result, ActorError> + where + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, + { + let (slot, rx) = reply_slot(crate::reply::NEVER_TICKS, Some(&self.replies), false); + let envelope: Box + Send> = Box::new(InTurnEnvelope { + f: Box::new(f), + slot, + }); + // Past the death point the actor is gone (E4), as for `enqueue`. + if self.replies.refuses_admission() { + return Err(ActorError::NoProc); + } + self.sender + .send(MailboxItem::Message(envelope)) + .map_err(|_| ActorError::NoProc)?; + Ok(rx) + } + /// Send a request and block until the reply arrives, with a custom timeout. pub fn request_with_timeout( &self, @@ -646,13 +1169,10 @@ impl ActorRef { M: Message, { let observation = crate::instrumentation::RequestObservation::new(); - let result = match self.request_raw(msg) { + let result = match self.send_request_with(msg, crate::reply::clock::after(duration), false) + { Err(error) => Err(error), - Ok(rx) => match rx.recv_timeout(duration) { - Ok(result) => Ok(result), - Err(RecvTimeoutError::Timeout) => Err(ActorError::RequestTimeout), - Err(RecvTimeoutError::Disconnected) => Err(ActorError::ActorStopped), - }, + Ok(rx) => rx.recv(), }; observation.finish(&result); result @@ -667,19 +1187,25 @@ impl ActorRef { Arc::new(self.clone()) } - /// Get a `Context` from this ref, for timer setup or stream listeners. + /// Get a `Context` from this ref, for timer setup or stream listeners + /// from outside the actor. pub fn context(&self) -> Context { Context::from_ref(self) } /// Block until the actor has fully stopped (including `stopped()` callback). + /// + /// Returns when cleanup completes: the actor died and its cancelled jobs + /// were joined (bounded by the cleanup deadline). Use + /// [`wait_exit`](Self::wait_exit) to observe death alone. pub fn join(&self) { let _ = self.wait_exit(); + self.cleaned.wait(); } - /// Poll the exit reason. Returns `None` if the actor is still running. - /// A completion channel closed without a reason reports [`ExitReason::Kill`], - /// matching the wait methods (for example, after runtime shutdown). + /// Poll the exit reason. Returns `None` until the exit is published. + /// A completion channel closed without a reason reports + /// [`ExitReason::Killed`], matching the wait methods. pub fn exit_reason(&self) -> Option { crate::child_handle::poll_exit_reason(&self.completion) } @@ -711,21 +1237,10 @@ impl ActorRef { impl From> for ChildHandle { fn from(actor_ref: ActorRef) -> Self { let failures = actor_ref.sender.failures(); - let sender = actor_ref.sender.clone(); - let send_exit: SendExitFn = Arc::new(move |exit: Exit| { - let envelope: Box + Send> = Box::new(ExitEnvelope { exit }); - sender - .send(MailboxItem::Message(envelope)) - .map_err(|_| ActorError::ActorStopped) - }); - ChildHandle::from_threads( - actor_ref.id, - actor_ref.cancellation_token, - actor_ref.completion, - actor_ref.trap_exit, + ChildHandle::new( + actor_ref.signal, + Completion::Threads(actor_ref.completion), actor_ref.links, - actor_ref.linked_reason, - send_exit, ) .with_mailbox_failures(failures) } @@ -741,8 +1256,20 @@ where ActorRef::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - ActorRef::request_raw(self, msg) + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request(self, msg, deadline) + } + + fn send_request_timed( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + ActorRef::send_request_with(self, msg, deadline.ticks(), false) } } @@ -757,10 +1284,20 @@ impl ActorRef { register_stop_wakeup(&cancellation_token, tx.clone()); let (completion_tx, completion) = spawned_rt::sync::watch::channel(None); let id = ActorId::next(); - let monitors: MonitorTable = Arc::new(Mutex::new(Some(HashMap::new()))); + let monitors: MonitorTable = Arc::new(crate::monitor::fence::open()); let trap_exit = new_trap_exit_flag(); let links = new_link_table(); let linked_reason = new_linked_exit_reason(); + // Phase 0: signals that arrive before started() returns are held. + link::lock_request(&linked_reason).begin_starting(); + let signal = signal_target::( + id, + parent.as_ref().map(ChildHandle::id), + &tx, + trap_exit.clone(), + linked_reason.clone(), + cancellation_token.clone(), + ); let actor_ref = ActorRef { id, @@ -771,7 +1308,14 @@ impl ActorRef { trap_exit: trap_exit.clone(), links: links.clone(), linked_reason: linked_reason.clone(), + signal: signal.clone(), mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(id, A::MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new( + id, + A::MAX_NATIVE_JOBS.unwrap_or_else(crate::reply::jobs::default_max_native_jobs), + )), + cleaned: Arc::default(), }; let ctx = Context { @@ -783,12 +1327,19 @@ impl ActorRef { trap_exit, links: links.clone(), linked_reason: linked_reason.clone(), + signal, mailbox: actor_ref.mailbox.clone(), + replies: actor_ref.replies.clone(), + jobs: actor_ref.jobs.clone(), + cleaned: actor_ref.cleaned.clone(), }; + let parent_id = parent.as_ref().map_or(0, |p| p.id().raw()); if let Some(parent) = parent { actor_ref.context().link(&parent); } + // Recorded from here: the start_linked link is the model's initial one. + crate::lifecycle_trace::life_spawn(id.raw(), parent_id); let cleanup_token = cancellation_token.clone(); let cleanup = crate::actor_cleanup::ActorCleanup::new( @@ -797,33 +1348,61 @@ impl ActorRef { linked_reason, actor_ref.monitors.clone(), completion_tx, - ExitReason::Panic("unexpected framework panic".into()), move || cleanup_token.cancel(), ) - .with_mailbox_failures(actor_ref.sender.failures()); - let startup = - crate::actor_cleanup::ActorStartup::new((actor, ctx, rx, cancellation_token), cleanup); + .with_mailbox_failures(actor_ref.sender.failures()) + .with_replies(actor_ref.replies.clone()) + .with_jobs(actor_ref.jobs.clone()); + let startup = crate::actor_cleanup::ActorStartup::new( + (actor, ctx, rx, cancellation_token), + cleanup, + discharge_startup::, + ); + let replies = actor_ref.replies.clone(); + let jobs = actor_ref.jobs.clone(); + let cleaned = crate::reply::jobs::CleanedGuard(actor_ref.cleaned.clone()); let _thread_handle = rt::spawn(move || { + let _cleaned = cleaned; let ((mut actor, ctx, mut rx, cancellation_token), mut cleanup) = startup.into_parts(); - let primary = match catch_unwind(AssertUnwindSafe(|| { + let request = ctx.linked_reason.clone(); + let discarded = match catch_unwind(AssertUnwindSafe(|| { run_actor(&mut actor, ctx, &mut rx, cancellation_token) })) { - Ok(reason) => Ok(reason), + Ok(discarded) => discarded, Err(payload) => { let message = panic_message(&*payload); crate::mailbox::dispose_panic(payload); - Err(ExitReason::Panic(format!( + link::lock_request(&request).panic(ExitReason::Panic(format!( "actor lifecycle unwound: {message}" - ))) + ))); + None } }; - let outcome = crate::actor_cleanup::finish_actor_state(actor, primary); - // State destructors may still send through a retained context. Keep - // the receiver alive until state is gone, then drain before publish. + // The death point D, only now that `stopped()` returned (or was + // skipped): the reason becomes final, the link table closes, job + // admission and the registry close (every open token is + // answered), then the mailbox refuses admission and its queued + // requests (and one received after the stop) are answered, all + // before any destructor. + cleanup.die(); + let queued = answer_mailbox(&mut rx, discarded, Some(&replies)); + // Every caller is answered: wake them, then destroy (queued + // payloads, then state). + cleanup.discharged(); + rx.discard_all(queued); drop(rx); - match outcome { - Ok(reason) => cleanup.complete(reason), - Err(reason) => cleanup.complete_secondary(reason), + // Destruction: a destructor panic is a cleanup failure only. + if let Some(failure) = crate::actor_cleanup::finish_actor_state(actor) { + cleanup.cleanup_failed(failure); + } + // Publication (wait_exit, monitors, link notices) before the job join. + drop(cleanup); + // Job cleanup (ยง5 step 4), after death: join the cancelled jobs + // that signal completion, bounded by the cleanup deadline. + if jobs.running() > 0 { + let remaining = replies.cleanup_remaining(DEFAULT_STOP_DEADLINE); + let unfinished = jobs.join_blocking(std::time::Instant::now() + remaining); + crate::reply::jobs::report_unfinished(jobs.owner(), unfinished); } }); @@ -831,12 +1410,14 @@ impl ActorRef { } } +/// Returns an envelope received after the stop (undispatched; answered at +/// the death point). fn run_actor( actor: &mut A, ctx: Context, rx: &mut mpsc::Receiver>, cancellation_token: CancellationToken, -) -> ExitReason { +) -> Option + Send>> { run_actor_receiving( actor, ctx, @@ -853,48 +1434,42 @@ fn run_actor_receiving( ctx: Context, mut receive: impl FnMut() -> Result, std::sync::mpsc::RecvTimeoutError>, cancellation_token: CancellationToken, -) -> ExitReason { +) -> Option + Send>> { let mut discarded = None; + let request = ctx.linked_reason.clone(); + let trace_id = ctx.id.raw(); let _monitor_cleanup = MonitorCleanup(ctx.monitors.clone()); let start_result = catch_unwind(AssertUnwindSafe(|| { fail_point!("actor.before_started"); - actor.started(&ctx); + crate::lifecycle_trace::with_actor(trace_id, || actor.started(&ctx)); })); if let Err(panic) = start_result { let msg = panic_message(&*panic); crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in started() callback: {msg}"); }); + // No stopped() after a failed start (L5). + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("started") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in started(): {msg}"))); + }); cancellation_token.cancel(); - return ExitReason::Panic(format!("panic in started(): {msg}")); + return None; } - - if cancellation_token.is_cancelled() { - if let Err(panic) = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_stopped"); - actor.stopped(&ctx) - })) { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); - }); - return ExitReason::Panic(format!("panic in stopped(): {msg}")); - } - return ExitReason::Normal; + // Phase 0 ends: resolve signals held during started() with the trap + // flag read now. + if link::lock_request(&request).end_starting(ctx.trap_exit.load(Ordering::Acquire)) { + cancellation_token.cancel(); } - let mut exit_reason = ExitReason::Normal; - - loop { + while !cancellation_token.is_cancelled() { let msg = match receive() { Ok(MailboxItem::Message(msg)) => Some(msg), Ok(MailboxItem::Shutdown) => None, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => { - if cancellation_token.is_cancelled() { - break; - } - continue; - } + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => continue, Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => None, }; match msg { @@ -904,7 +1479,19 @@ fn run_actor_receiving( // A stop observed after receive wins over dispatch, whether // the channel waited or was buffered. Defer the message drop. - if cancellation_token.is_cancelled() { + // The check and the trace event are one lifecycle-trace step. + let stopped = crate::lifecycle_trace::life_section(trace_id, || { + if cancellation_token.is_cancelled() { + return true; + } + #[cfg(feature = "lifecycle-trace")] + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Dispatch", trace_id) + .k(envelope.trace_kind()) + }); + false + }); + if stopped { discarded = Some(envelope); break; } @@ -912,9 +1499,11 @@ fn run_actor_receiving( let observation = envelope .observation() .map(|observation| observation.begin()); + ctx.replies.begin_turn(); let result = catch_unwind(AssertUnwindSafe(|| { - envelope.handle(actor, &ctx); + crate::lifecycle_trace::with_actor(trace_id, || envelope.handle(actor, &ctx)); })); + ctx.replies.end_turn(result.is_err()); #[cfg(feature = "metrics")] if let Some(observation) = observation { observation.finish(result.is_err()); @@ -924,10 +1513,15 @@ fn run_actor_receiving( crate::actor_cleanup::log_safely(|| { tracing::error!("Panic in message handler: {msg}"); }); - exit_reason = ExitReason::Panic(format!("panic in handler: {msg}")); - break; - } - if cancellation_token.is_cancelled() { + // A panic begins exit, or replaces the requested reason + // unless a kill was received. + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("turn") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in handler: {msg}"))); + }); break; } } @@ -936,25 +1530,60 @@ fn run_actor_receiving( } cancellation_token.cancel(); - let stop_result = catch_unwind(AssertUnwindSafe(|| { - fail_point!("actor.before_stopped"); - actor.stopped(&ctx); - })); - if let Err(panic) = stop_result { - let msg = panic_message(&*panic); - crate::actor_cleanup::log_safely(|| { - tracing::error!("Panic in stopped() callback: {msg}"); + ctx.replies.exiting(); + // Payload drops are held until every caller is answered (DropFailures). + ctx.sender.failures().hold(); + ctx.jobs.cancel_all(); + // stopped() is skipped when a kill, an untrapped exit signal or link + // notice began the exit (decision 2); one that already began finishes. + let (run_hook, skipped) = crate::lifecycle_trace::life_section(trace_id, || { + let mut req = link::lock_request(&request); + let run = req.begin_hook(); + let skipped = (!run).then(|| (req.skip_cause(), req.reason().cloned())); + drop(req); + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Hook", trace_id).x(if run { + "run" + } else { + "skip" + }) }); - if !exit_reason.is_abnormal() { - exit_reason = ExitReason::Panic(format!("panic in stopped(): {msg}")); + (run, skipped) + }); + if let Some((cause, reason)) = skipped { + crate::actor_cleanup::log_stopped_skipped( + ctx.id, + std::any::type_name::(), + cause, + reason.as_ref(), + ); + } + if run_hook { + let stop_result = catch_unwind(AssertUnwindSafe(|| { + fail_point!("actor.before_stopped"); + crate::lifecycle_trace::with_actor(trace_id, || actor.stopped(&ctx)); + })); + if let Err(panic) = stop_result { + let msg = panic_message(&*panic); + crate::actor_cleanup::log_safely(|| { + tracing::error!("Panic in stopped() callback: {msg}"); + }); + crate::lifecycle_trace::life_section(trace_id, || { + crate::lifecycle_trace::life_emit(trace_id, || { + crate::lifecycle_trace::LifeEvent::new("Panic", trace_id).x("stopped") + }); + link::lock_request(&request) + .panic(ExitReason::Panic(format!("panic in stopped(): {msg}"))); + }); } } // Message destructors and reply cancellation may re-enter monitor state. - // Preserve stopped() -> monitor cleanup -> undelivered-message drop. + // Preserve stopped() -> monitor cleanup -> undelivered-message drop (the + // envelope received after the stop is answered at the death point, then + // dropped with the mailbox). drop(_monitor_cleanup); - ctx.sender.failures().discard(discarded); - exit_reason + discarded } // --------------------------------------------------------------------------- @@ -968,9 +1597,10 @@ pub trait ActorStart: Actor { ActorRef::spawn(self, None) } - /// Register the bidirectional link before the child can run `started()`. - /// A closed parent is observed under its link-state mutex; an abnormal - /// parent exit cancels the child cooperatively before message dispatch. + /// Start linked to `parent_ctx`'s actor, which becomes this instance's + /// parent (E3). The link is registered before `started()`. The parent's + /// exit ends a trapping child normally (its `stopped()` runs; it never + /// reaches `exit_received`), a non-trapping child as any link notice. fn start_linked(self, parent_ctx: &Context

) -> ActorRef { ActorRef::spawn(self, Some(parent_ctx.actor_ref().child_handle())) } @@ -1031,245 +1661,385 @@ mod tests { impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, _msg: GetCount, _ctx: &Context) -> u64 { - self.count + fn handle( + &mut self, + _msg: GetCount, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.count); } } impl Handler for Counter { - fn handle(&mut self, _msg: Increment, _ctx: &Context) -> u64 { - self.count += 1; - self.count + fn handle( + &mut self, + _msg: Increment, + reply: crate::reply::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __reply_value: u64 = { + self.count += 1; + self.count + }; + reply.reply(__reply_value); } } impl Handler for Counter { - fn handle(&mut self, _msg: StopCounter, ctx: &Context) -> u64 { - ctx.stop(); - self.count + fn handle( + &mut self, + _msg: StopCounter, + reply: crate::reply::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); } } #[test] fn basic_send_and_request() { - let actor = Counter { count: 0 }.start(); - assert_eq!(actor.request(GetCount).unwrap(), 0); - assert_eq!(actor.request(Increment).unwrap(), 1); - // Same mailbox: the request is handled after the send. - actor.send(Increment).unwrap(); - assert_eq!(actor.request(GetCount).unwrap(), 2); - actor.request(StopCounter).unwrap(); + crate::test_wait::bounded( + concat!(module_path!(), "::basic_send_and_request"), + move || { + let actor = Counter { count: 0 }.start(); + assert_eq!(actor.request(GetCount).unwrap(), 0); + assert_eq!(actor.request(Increment).unwrap(), 1); + // Same mailbox: the request is handled after the send. + actor.send(Increment).unwrap(); + assert_eq!(actor.request(GetCount).unwrap(), 2); + actor.request(StopCounter).unwrap(); + }, + ); } #[test] fn join_waits_for_completion() { - struct SlowStop; - struct StopSlow; - impl Message for StopSlow { - type Result = (); - } - impl Actor for SlowStop { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(300)); - } - } - impl Handler for SlowStop { - fn handle(&mut self, _msg: StopSlow, ctx: &Context) { - ctx.stop(); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::join_waits_for_completion"), + move || { + struct SlowStop; + struct StopSlow; + impl Message for StopSlow { + type Result = (); + } + impl Actor for SlowStop { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(300)); + } + } + impl Handler for SlowStop { + fn handle( + &mut self, + _msg: StopSlow, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = SlowStop.start(); - actor.send(StopSlow).unwrap(); - actor.join(); - // If join() returned, stopped() has completed + let actor = SlowStop.start(); + actor.send(StopSlow).unwrap(); + actor.join(); + // If join() returned, stopped() has completed + }, + ); } #[test] fn join_multiple_callers() { - struct SlowStop2; - struct StopSlow2; - impl Message for StopSlow2 { - type Result = (); - } - impl Actor for SlowStop2 { - fn stopped(&mut self, _ctx: &Context) { - rt::sleep(Duration::from_millis(200)); - } - } - impl Handler for SlowStop2 { - fn handle(&mut self, _msg: StopSlow2, ctx: &Context) { - ctx.stop(); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::join_multiple_callers"), + move || { + struct SlowStop2; + struct StopSlow2; + impl Message for StopSlow2 { + type Result = (); + } + impl Actor for SlowStop2 { + fn stopped(&mut self, _ctx: &Context) { + rt::sleep(Duration::from_millis(200)); + } + } + impl Handler for SlowStop2 { + fn handle( + &mut self, + _msg: StopSlow2, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = SlowStop2.start(); - let a1 = actor.clone(); - let a2 = actor.clone(); - let t1 = thread::spawn(move || { - a1.join(); - 1u32 - }); - let t2 = thread::spawn(move || { - a2.join(); - 2u32 - }); - actor.send(StopSlow2).unwrap(); - assert_eq!(t1.join().unwrap(), 1); - assert_eq!(t2.join().unwrap(), 2); + let actor = SlowStop2.start(); + let a1 = actor.clone(); + let a2 = actor.clone(); + let t1 = thread::spawn(move || { + a1.join(); + 1u32 + }); + let t2 = thread::spawn(move || { + a2.join(); + 2u32 + }); + actor.send(StopSlow2).unwrap(); + assert_eq!(t1.join().unwrap(), 1); + assert_eq!(t2.join().unwrap(), 2); + }, + ); } #[test] fn panic_in_started_stops_actor() { - struct PanicOnStart; - struct PingThread; - impl Message for PingThread { - type Result = (); - } - impl Actor for PanicOnStart { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicOnStart { - fn handle(&mut self, _msg: PingThread, _ctx: &Context) {} - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_started_stops_actor"), + move || { + struct PanicOnStart; + struct PingThread; + impl Message for PingThread { + type Result = (); + } + impl Actor for PanicOnStart { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicOnStart { + fn handle( + &mut self, + _msg: PingThread, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicOnStart.start(); - actor.join(); - let result = actor.send(PingThread); - assert!(result.is_err()); + let actor = PanicOnStart.start(); + actor.join(); + let result = actor.send(PingThread); + assert!(result.is_err()); + }, + ); } #[test] fn panic_in_handler_stops_actor() { - struct PanicOnMsg; - struct ExplodeThread; - impl Message for ExplodeThread { - type Result = (); - } - struct CheckThread; - impl Message for CheckThread { - type Result = u32; - } - impl Actor for PanicOnMsg {} - impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: ExplodeThread, _ctx: &Context) { - panic!("boom in handler"); - } - } - impl Handler for PanicOnMsg { - fn handle(&mut self, _msg: CheckThread, _ctx: &Context) -> u32 { - 42 - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_handler_stops_actor"), + move || { + struct PanicOnMsg; + struct ExplodeThread; + impl Message for ExplodeThread { + type Result = (); + } + struct CheckThread; + impl Message for CheckThread { + type Result = u32; + } + impl Actor for PanicOnMsg {} + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: ExplodeThread, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } + impl Handler for PanicOnMsg { + fn handle( + &mut self, + _msg: CheckThread, + reply: crate::reply::PendingReply<'_, u32>, + _ctx: &Context, + ) { + reply.reply(42); + } + } - let actor = PanicOnMsg.start(); - actor.send(ExplodeThread).unwrap(); - actor.join(); - let result = actor.request(CheckThread); - assert!(result.is_err()); + let actor = PanicOnMsg.start(); + actor.send(ExplodeThread).unwrap(); + actor.join(); + let result = actor.request(CheckThread); + assert!(result.is_err()); + }, + ); } #[test] fn panic_in_stopped_still_completes() { - struct PanicOnStop; - struct StopMeThread; - impl Message for StopMeThread { - type Result = (); - } - impl Actor for PanicOnStop { - fn stopped(&mut self, _ctx: &Context) { - panic!("boom in stopped"); - } - } - impl Handler for PanicOnStop { - fn handle(&mut self, _msg: StopMeThread, ctx: &Context) { - ctx.stop(); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::panic_in_stopped_still_completes"), + move || { + struct PanicOnStop; + struct StopMeThread; + impl Message for StopMeThread { + type Result = (); + } + impl Actor for PanicOnStop { + fn stopped(&mut self, _ctx: &Context) { + panic!("boom in stopped"); + } + } + impl Handler for PanicOnStop { + fn handle( + &mut self, + _msg: StopMeThread, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.stop(); + reply.reply(()); + } + } - let actor = PanicOnStop.start(); - actor.send(StopMeThread).unwrap(); - actor.join(); + let actor = PanicOnStop.start(); + actor.send(StopMeThread).unwrap(); + actor.join(); + }, + ); } #[test] fn recipient_type_erasure() { - let actor = Counter { count: 42 }.start(); - let recipient: Recipient = actor.recipient(); - let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); - assert_eq!(result, 42); + crate::test_wait::bounded( + concat!(module_path!(), "::recipient_type_erasure"), + move || { + let actor = Counter { count: 42 }.start(); + let recipient: Recipient = actor.recipient(); + let result = request(&*recipient, GetCount, Duration::from_secs(5)).unwrap(); + assert_eq!(result, 42); + }, + ); } #[test] fn send_message_on_delivers() { - let actor = Counter { count: 0 }.start(); - let ctx = actor.context(); - send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); - let count = - crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))).unwrap(); - assert_eq!(count, 1); + crate::test_wait::bounded( + concat!(module_path!(), "::send_message_on_delivers"), + move || { + let actor = Counter { count: 0 }.start(); + let ctx = actor.context(); + send_message_on(ctx, || rt::sleep(Duration::from_millis(10)), Increment); + let count = + crate::test_wait::wait_for(|| actor.request(GetCount), |c| matches!(c, Ok(1))) + .unwrap(); + assert_eq!(count, 1); + }, + ); } // --- ExitReason tests --- #[test] fn exit_reason_normal_on_clean_stop() { - let actor = Counter { count: 0 }.start(); - actor.request(StopCounter).unwrap(); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Normal)); + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_normal_on_clean_stop"), + move || { + let actor = Counter { count: 0 }.start(); + actor.request(StopCounter).unwrap(); + let reason = actor.wait_exit(); + assert!(matches!(reason, ExitReason::Normal)); + }, + ); } #[test] fn exit_reason_panic_in_started() { - struct PanicStartThread; - struct PingThread2; - impl Message for PingThread2 { - type Result = (); - } - impl Actor for PanicStartThread { - fn started(&mut self, _ctx: &Context) { - panic!("boom in started"); - } - } - impl Handler for PanicStartThread { - fn handle(&mut self, _msg: PingThread2, _ctx: &Context) {} - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_started"), + move || { + struct PanicStartThread; + struct PingThread2; + impl Message for PingThread2 { + type Result = (); + } + impl Actor for PanicStartThread { + fn started(&mut self, _ctx: &Context) { + panic!("boom in started"); + } + } + impl Handler for PanicStartThread { + fn handle( + &mut self, + _msg: PingThread2, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } + } - let actor = PanicStartThread.start(); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started"))); + let actor = PanicStartThread.start(); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in started")) + ); + }, + ); } #[test] fn exit_reason_panic_in_handler() { - struct PanicHandlerThread; - struct ExplodeThread2; - impl Message for ExplodeThread2 { - type Result = (); - } - impl Actor for PanicHandlerThread {} - impl Handler for PanicHandlerThread { - fn handle(&mut self, _msg: ExplodeThread2, _ctx: &Context) { - panic!("boom in handler"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::exit_reason_panic_in_handler"), + move || { + struct PanicHandlerThread; + struct ExplodeThread2; + impl Message for ExplodeThread2 { + type Result = (); + } + impl Actor for PanicHandlerThread {} + impl Handler for PanicHandlerThread { + fn handle( + &mut self, + _msg: ExplodeThread2, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom in handler"); + } + } + } - let actor = PanicHandlerThread.start(); - let _ = actor.send(ExplodeThread2); - let reason = actor.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler"))); + let actor = PanicHandlerThread.start(); + let _ = actor.send(ExplodeThread2); + let reason = actor.wait_exit(); + assert!( + matches!(reason, ExitReason::Panic(ref msg) if msg.contains("boom in handler")) + ); + }, + ); } #[test] fn exit_reason_poll_returns_none_while_running() { - let actor = Counter { count: 0 }.start(); - assert!(actor.exit_reason().is_none()); - actor.request(StopCounter).unwrap(); - actor.join(); - assert!(actor.exit_reason().is_some()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::exit_reason_poll_returns_none_while_running" + ), + move || { + let actor = Counter { count: 0 }.start(); + assert!(actor.exit_reason().is_none()); + actor.request(StopCounter).unwrap(); + actor.join(); + assert!(actor.exit_reason().is_some()); + }, + ); } // --- Monitor tests --- @@ -1291,30 +2061,70 @@ mod tests { impl Message for CallDemonitor { type Result = (); } + /// Monitor, then demonitor in the same turn. + struct MonitorThenDemonitor(crate::ChildHandle); + impl Message for MonitorThenDemonitor { + type Result = (); + } impl Actor for Watcher {} impl Handler for Watcher { - fn handle(&mut self, msg: StartMonitor, ctx: &Context) -> crate::monitor::MonitorRef { - ctx.monitor(&msg.0) + fn handle( + &mut self, + msg: StartMonitor, + reply: crate::reply::PendingReply<'_, crate::monitor::MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); + } + } + + impl Handler for Watcher { + fn handle( + &mut self, + msg: MonitorThenDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { + let monitor = ctx.monitor(&msg.0); + ctx.demonitor(monitor); + reply.reply(()); } } impl Handler for Watcher { - fn handle(&mut self, msg: CallDemonitor, ctx: &Context) { + fn handle( + &mut self, + msg: CallDemonitor, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Watcher { - fn handle(&mut self, msg: crate::monitor::Down, _ctx: &Context) { + fn handle( + &mut self, + msg: crate::monitor::Down, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.downs.lock().unwrap().push(msg); + reply.reply(()); } } impl Handler for Watcher { - fn handle(&mut self, _msg: GetDowns, _ctx: &Context) -> Vec { - self.downs.lock().unwrap().clone() + fn handle( + &mut self, + _msg: GetDowns, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.downs.lock().unwrap().clone()); } } @@ -1327,111 +2137,177 @@ mod tests { #[test] fn monitor_running_actor_delivers_down_on_exit() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_running_actor_delivers_down_on_exit" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].monitor_ref, monitor_ref); + assert!(matches!(downs[0].reason, ExitReason::Normal)); + }, + ); + } - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert_eq!(downs[0].monitor_ref, monitor_ref); - assert!(matches!(downs[0].reason, ExitReason::Normal)); + /// OTP `demonitor(Ref, [flush])`: a `Down` already in the mailbox + /// (here the `NoProc` a monitor of a dead target queues at once) is + /// never handled once `demonitor` returned. + #[test] + fn demonitor_flushes_a_queued_down() { + crate::test_wait::bounded( + concat!(module_path!(), "::demonitor_flushes_a_queued_down"), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let watcher = make_watcher(); + watcher + .request(MonitorThenDemonitor(target.child_handle())) + .unwrap(); + // FIFO: the queued Down is dispatched before this request. + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty(), "a flushed Down was handled: {downs:?}"); + }, + ); } #[test] fn monitor_already_dead_actor_delivers_down_immediately() { - let target = Counter { count: 0 }.start(); - target.request(StopCounter).unwrap(); - target.join(); - let target_handle = target.child_handle(); - - let watcher = make_watcher(); - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); + crate::test_wait::bounded( + concat!( + module_path!(), + "::monitor_already_dead_actor_delivers_down_immediately" + ), + move || { + let target = Counter { count: 0 }.start(); + target.request(StopCounter).unwrap(); + target.join(); + let target_handle = target.child_handle(); + + let watcher = make_watcher(); + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + }, + ); } #[test] fn demonitor_before_target_dies_suppresses_down() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); - watcher.request(CallDemonitor(monitor_ref)).unwrap(); - - target.request(StopCounter).unwrap(); - target.join(); - rt::sleep(Duration::from_millis(150)); - - let downs = watcher.request(GetDowns).unwrap(); - assert!(downs.is_empty()); + crate::test_wait::bounded( + concat!( + module_path!(), + "::demonitor_before_target_dies_suppresses_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let monitor_ref = watcher.request(StartMonitor(target_handle)).unwrap(); + watcher.request(CallDemonitor(monitor_ref)).unwrap(); + + target.request(StopCounter).unwrap(); + target.join(); + rt::sleep(Duration::from_millis(150)); + + let downs = watcher.request(GetDowns).unwrap(); + assert!(downs.is_empty()); + }, + ); } #[test] fn multiple_monitors_each_get_own_ref_and_down() { - let target = Counter { count: 0 }.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); - - let r1 = watcher - .request(StartMonitor(target_handle.clone())) - .unwrap(); - let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); - assert_ne!(r1, r2); - - target.request(StopCounter).unwrap(); - target.join(); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), - ) - .unwrap(); - assert_eq!(downs.len(), 2); - let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); - assert!(refs.contains(&r1)); - assert!(refs.contains(&r2)); + crate::test_wait::bounded( + concat!( + module_path!(), + "::multiple_monitors_each_get_own_ref_and_down" + ), + move || { + let target = Counter { count: 0 }.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); + + let r1 = watcher + .request(StartMonitor(target_handle.clone())) + .unwrap(); + let r2 = watcher.request(StartMonitor(target_handle)).unwrap(); + assert_ne!(r1, r2); + + target.request(StopCounter).unwrap(); + target.join(); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| d.len() >= 2), + ) + .unwrap(); + assert_eq!(downs.len(), 2); + let refs: Vec<_> = downs.iter().map(|d| d.monitor_ref).collect(); + assert!(refs.contains(&r1)); + assert!(refs.contains(&r2)); + }, + ); } #[test] fn monitor_observes_panic_reason() { - struct PanicMsg; - impl Message for PanicMsg { - type Result = (); - } - struct PanicMe; - impl Actor for PanicMe {} - impl Handler for PanicMe { - fn handle(&mut self, _msg: PanicMsg, _ctx: &Context) { - panic!("intentional panic"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::monitor_observes_panic_reason"), + move || { + struct PanicMsg; + impl Message for PanicMsg { + type Result = (); + } + struct PanicMe; + impl Actor for PanicMe {} + impl Handler for PanicMe { + fn handle( + &mut self, + _msg: PanicMsg, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("intentional panic"); + } + } + } - let target = PanicMe.start(); - let target_handle = target.child_handle(); - let watcher = make_watcher(); + let target = PanicMe.start(); + let target_handle = target.child_handle(); + let watcher = make_watcher(); - let _ = watcher.request(StartMonitor(target_handle)).unwrap(); - let _ = target.send(PanicMsg); + let _ = watcher.request(StartMonitor(target_handle)).unwrap(); + let _ = target.send(PanicMsg); - let downs = crate::test_wait::wait_for( - || watcher.request(GetDowns), - |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), - ) - .unwrap(); - assert_eq!(downs.len(), 1); - assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + let downs = crate::test_wait::wait_for( + || watcher.request(GetDowns), + |downs| downs.as_ref().map_or(true, |d| !d.is_empty()), + ) + .unwrap(); + assert_eq!(downs.len(), 1); + assert!(matches!(downs[0].reason, ExitReason::Panic(_))); + }, + ); } // --- Link tests --- @@ -1460,14 +2336,25 @@ mod tests { } impl Handler for TrapActor { - fn handle(&mut self, msg: LinkTo, ctx: &Context) { + fn handle( + &mut self, + msg: LinkTo, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for TrapActor { - fn handle(&mut self, _msg: GetExits, _ctx: &Context) -> Vec { - self.exits.lock().unwrap().clone() + fn handle( + &mut self, + _msg: GetExits, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.exits.lock().unwrap().clone()); } } @@ -1481,84 +2368,126 @@ mod tests { #[test] fn link_propagates_panic_to_non_trapping_peer_threads() { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { - panic!("boom"); - } - } + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_propagates_panic_to_non_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } - let a = make_trapper(false); - let b = make_trapper(false); - a.request(LinkTo(b.child_handle())).unwrap(); + let a = make_trapper(false); + let b = make_trapper(false); + a.request(LinkTo(b.child_handle())).unwrap(); - let _ = a.send(Boom); - let reason_a = a.wait_exit(); - let reason_b = b.wait_exit(); - assert!(matches!(reason_a, ExitReason::Panic(_))); - assert!(matches!(reason_b, ExitReason::Panic(_))); + let _ = a.send(Boom); + let reason_a = a.wait_exit(); + let reason_b = b.wait_exit(); + assert!(matches!(reason_a, ExitReason::Panic(_))); + assert!(matches!(reason_b, ExitReason::Panic(_))); + }, + ); } #[test] fn link_delivers_exit_to_trapping_peer_threads() { - struct Boom; - impl Message for Boom { - type Result = (); - } - impl Handler for TrapActor { - fn handle(&mut self, _msg: Boom, _ctx: &Context) { - panic!("boom"); - } - } - - let a = make_trapper(false); - let b = make_trapper(true); - b.request(LinkTo(a.child_handle())).unwrap(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::link_delivers_exit_to_trapping_peer_threads" + ), + move || { + struct Boom; + impl Message for Boom { + type Result = (); + } + impl Handler for TrapActor { + fn handle( + &mut self, + _msg: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("boom"); + } + } + } - let _ = a.send(Boom); - a.wait_exit(); - let exits = crate::test_wait::wait_for( - || b.request(GetExits), - |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), - ) - .unwrap(); - assert!(b.exit_reason().is_none()); - assert_eq!(exits.len(), 1); - assert_eq!(exits[0].from, a.id()); - assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + let a = make_trapper(false); + let b = make_trapper(true); + b.request(LinkTo(a.child_handle())).unwrap(); - let bh = b.child_handle(); - bh.stop(); - bh.wait_exit_blocking(); + let _ = a.send(Boom); + a.wait_exit(); + let exits = crate::test_wait::wait_for( + || b.request(GetExits), + |exits| exits.as_ref().map_or(true, |e| !e.is_empty()), + ) + .unwrap(); + assert!(b.exit_reason().is_none()); + assert_eq!(exits.len(), 1); + assert_eq!(exits[0].from, a.id()); + assert!(matches!(exits[0].reason, ExitReason::Panic(_))); + + let bh = b.child_handle(); + bh.stop(); + bh.wait_exit_blocking(); + }, + ); } #[test] fn start_linked_threads() { - struct Parent; - struct PanicParent; - impl Message for PanicParent { - type Result = (); - } - impl Actor for Parent {} - impl Handler for Parent { - fn handle(&mut self, _msg: PanicParent, _ctx: &Context) { - panic!("parent boom"); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::start_linked_threads"), + move || { + struct Parent; + struct PanicParent; + impl Message for PanicParent { + type Result = (); + } + impl Actor for Parent {} + impl Handler for Parent { + fn handle( + &mut self, + _msg: PanicParent, + _reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { + { + panic!("parent boom"); + } + } + } - struct Child; - impl Actor for Child {} + struct Child; + impl Actor for Child {} - let parent = Parent.start(); - let child = Child.start_linked(&parent.context()); + let parent = Parent.start(); + let child = Child.start_linked(&parent.context()); - let _ = parent.send(PanicParent); - parent.wait_exit(); - let reason = child.wait_exit(); - assert!(matches!(reason, ExitReason::Panic(_))); + let _ = parent.send(PanicParent); + parent.wait_exit(); + let reason = child.wait_exit(); + assert!(matches!(reason, ExitReason::Panic(_))); + }, + ); } } @@ -1568,24 +2497,39 @@ mod monitor_cleanup_regressions { struct Idle; impl Actor for Idle {} impl Handler for Idle { - fn handle(&mut self, _: Down, _: &Context) {} + fn handle( + &mut self, + _: Down, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn closed_monitor_table_rejects_late_registration() { - let observer = Idle.start(); - let target = Idle.start(); - // Model cleanup during an unexpected unwind before token cancellation. - drop(MonitorCleanup(observer.monitors.clone())); - let child = target.child_handle(); - let waiter = child.lifetime_probe(); - observer.context().monitor(&child); - drop(child); - assert_eq!(waiter.strong_count(), 0); - assert!(observer.monitors.lock().unwrap().is_none()); - observer.child_handle().stop(); - target.child_handle().stop(); - observer.join(); - target.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::closed_monitor_table_rejects_late_registration" + ), + move || { + let observer = Idle.start(); + let target = Idle.start(); + // Model cleanup during an unexpected unwind before token cancellation. + drop(MonitorCleanup(observer.monitors.clone())); + let child = target.child_handle(); + let waiter = child.lifetime_probe(); + observer.context().monitor(&child); + drop(child); + assert_eq!(waiter.strong_count(), 0); + assert!(observer.monitors.lock().unwrap().is_none()); + observer.child_handle().stop(); + target.child_handle().stop(); + observer.join(); + target.join(); + }, + ); } fn check_cleanup(stop_owner: bool) { let target = Idle.start(); @@ -1639,9 +2583,14 @@ mod early_stop_regression { } #[test] fn panic_during_early_stop_is_reported() { - let actor = EarlyStop.start(); - assert!( - matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) + crate::test_wait::bounded( + concat!(module_path!(), "::panic_during_early_stop_is_reported"), + move || { + let actor = EarlyStop.start(); + assert!( + matches!(actor.wait_exit(), ExitReason::Panic(message) if message.contains("cleanup failed")) + ); + }, ); } } @@ -1678,74 +2627,116 @@ mod idle_stop_regression { type Result = (); } impl Handler for Counter { - fn handle(&mut self, _: Increment, _: &Context) { + fn handle( + &mut self, + _: Increment, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } #[test] fn stop_while_receiving_does_not_dispatch_a_later_request() { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - let (_, completion_rx) = spawned_rt::sync::watch::channel(None); - let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); - let events = Arc::new(Mutex::new(Vec::new())); - let ctx = Context { - id: ActorId::next(), - sender: tx.clone(), - cancellation_token: token.clone(), - completion: completion_rx, - monitors: monitors.clone(), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - }; - let count = Arc::new(AtomicUsize::new(0)); - let seen = count.clone(); - let entered = Arc::new(Barrier::new(2)); - let release = Arc::new(Barrier::new(2)); - let (e, r) = (entered.clone(), release.clone()); - let cancel = token.clone(); - let trace = events.clone(); - let actor = std::thread::spawn(move || { - let mut first = true; - run_actor_receiving( - &mut Counter(seen, trace), - ctx, - move || { - // Pause the real receive boundary after the started/cancel - // checks; the standard channel still carries the envelope. - if first { - first = false; - e.wait(); - r.wait(); - } - rx.recv_timeout(Duration::from_secs(5)) - }, - cancel, - ) - }); - entered.wait(); - let (reply, response) = oneshot::channel(); - tx.send(MailboxItem::Message(Box::new(MessageEnvelope { - msg: Increment(events.clone(), monitors), - tx: Some(reply), - #[cfg(feature = "metrics")] - observation: None, - }))) - .unwrap(); - token.cancel(); - release.wait(); - assert_eq!(actor.join().unwrap(), ExitReason::Normal); - assert_eq!( - count.load(Ordering::SeqCst), - 0, - "thread actor dispatched work after idle stop" - ); - assert!(response.recv().is_err()); - assert_eq!( - *events.lock().unwrap(), - ["stopped", "message-dropped", "actor-dropped"] + crate::test_wait::bounded( + concat!( + module_path!(), + "::stop_while_receiving_does_not_dispatch_a_later_request" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + let (_, completion_rx) = spawned_rt::sync::watch::channel(None); + let monitors = Arc::new(Mutex::new(Some(HashMap::new()))); + let events = Arc::new(Mutex::new(Vec::new())); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion: completion_rx, + monitors: monitors.clone(), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let count = Arc::new(AtomicUsize::new(0)); + let seen = count.clone(); + let entered = Arc::new(Barrier::new(2)); + let release = Arc::new(Barrier::new(2)); + let (e, r) = (entered.clone(), release.clone()); + let cancel = token.clone(); + let trace = events.clone(); + let actor = std::thread::spawn(move || { + let mut first = true; + let mut counter = Counter(seen, trace); + // The envelope received after the stop comes back undispatched; + // the death point answers it and destroys it before the state. + let discarded = run_actor_receiving( + &mut counter, + ctx, + move || { + // Pause the real receive boundary after the started/cancel + // checks; the standard channel still carries the envelope. + if first { + first = false; + e.wait(); + r.wait(); + } + rx.recv_timeout(Duration::from_secs(5)) + }, + cancel, + ); + drop(discarded); + drop(counter); + }); + entered.wait(); + let (slot, response) = reply_slot(crate::reply::NEVER_TICKS, None, false); + tx.send(MailboxItem::Message(Box::new(MessageEnvelope { + msg: Increment(events.clone(), monitors), + slot, + #[cfg(feature = "metrics")] + observation: None, + #[cfg(feature = "lifecycle-trace")] + from: 0, + }))) + .unwrap(); + token.cancel(); + release.wait(); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + assert_eq!( + count.load(Ordering::SeqCst), + 0, + "thread actor dispatched work after idle stop" + ); + assert!(response.recv().is_err()); + assert_eq!( + *events.lock().unwrap(), + ["stopped", "message-dropped", "actor-dropped"] + ); + }, ); } } @@ -1766,7 +2757,14 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Noop, _: &Context) {} + fn handle( + &mut self, + _: Noop, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] @@ -1799,42 +2797,71 @@ mod stop_wakeup_regression { #[test] fn cancellation_wakes_the_real_blocking_receive_boundary() { - let (tx, rx) = mpsc::channel(); - let token = CancellationToken::new(); - register_stop_wakeup(&token, tx.clone()); - let (_, completion) = spawned_rt::sync::watch::channel(None); - let ctx = Context { - id: ActorId::next(), - sender: tx, - cancellation_token: token.clone(), - completion, - monitors: Arc::new(Mutex::new(Some(HashMap::new()))), - trap_exit: new_trap_exit_flag(), - links: new_link_table(), - linked_reason: new_linked_exit_reason(), - mailbox: crate::instrumentation::Mailbox::default(), - }; - let (entered_tx, entered) = mpsc::channel(); - let (stopped_tx, stopped) = mpsc::channel(); - let cancel = token.clone(); - let actor = std::thread::spawn(move || { - run_actor_receiving( - &mut Idle(stopped_tx), - ctx, - move || { - let _ = entered_tx.send(()); - rx.recv() - .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) - }, - cancel, - ) - }); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - token.cancel(); - stopped - .recv_timeout(Duration::from_secs(2)) - .expect("cancel did not wake recv"); - assert_eq!(actor.join().unwrap(), ExitReason::Normal); + crate::test_wait::bounded( + concat!( + module_path!(), + "::cancellation_wakes_the_real_blocking_receive_boundary" + ), + move || { + let (tx, rx) = mpsc::channel(); + let token = CancellationToken::new(); + register_stop_wakeup(&token, tx.clone()); + let (_, completion) = spawned_rt::sync::watch::channel(None); + let (id, trap, request) = ( + ActorId::next(), + new_trap_exit_flag(), + new_linked_exit_reason(), + ); + let ctx = Context { + id, + sender: tx.clone(), + cancellation_token: token.clone(), + completion, + monitors: Arc::new(Mutex::new(Some(HashMap::new()))), + trap_exit: trap.clone(), + links: new_link_table(), + linked_reason: request.clone(), + signal: signal_target::<_>( + id, + None, + &tx, + trap.clone(), + request.clone(), + token.clone(), + ), + mailbox: crate::instrumentation::Mailbox::default(), + replies: Registry::new(ActorId::next(), DEFAULT_MAX_OUTSTANDING), + jobs: Arc::new(crate::reply::jobs::JobSet::new(ActorId::next(), 1)), + cleaned: Arc::default(), + }; + let (entered_tx, entered) = mpsc::channel(); + let (stopped_tx, stopped) = mpsc::channel(); + let cancel = token.clone(); + let actor = std::thread::spawn(move || { + run_actor_receiving( + &mut Idle(stopped_tx), + ctx, + move || { + let _ = entered_tx.send(()); + rx.recv() + .map_err(|_| std::sync::mpsc::RecvTimeoutError::Disconnected) + }, + cancel, + ) + }); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + token.cancel(); + stopped + .recv_timeout(Duration::from_secs(2)) + .expect("cancel did not wake recv"); + actor.join().unwrap(); + assert_eq!( + link::lock_request(&request).reason(), + None, + "a plain stop request" + ); + }, + ); } struct Crash; @@ -1844,49 +2871,70 @@ mod stop_wakeup_regression { struct Failing; impl Actor for Failing {} impl Handler for Failing { - fn handle(&mut self, _: Crash, _: &Context) { - panic!("linked wakeup regression"); + fn handle( + &mut self, + _: Crash, + _reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("linked wakeup regression"); + } } } #[test] fn every_stop_route_wakes_an_idle_thread() { - for route in 0..4 { - let (stopped_tx, stopped) = mpsc::channel(); - let actor = Idle(stopped_tx).start(); - // A reply ensures startup is complete before the cancellation race. - actor.request(Noop).unwrap(); - let failing = if route == 3 { - let peer = Failing.start(); - actor.context().link(&peer.child_handle()); - Some(peer) - } else { - None - }; - match route { - 0 => actor.context().stop(), - 1 => actor.child_handle().stop(), - 2 => actor.cancellation_token.cancel(), - _ => failing.as_ref().unwrap().send(Crash).unwrap(), - } - let result = stopped.recv_timeout(Duration::from_secs(2)); - // A failed test must not hang cleanup on a broken wakeup callback. - if result.is_err() { - actor.child_handle().stop(); - let _ = actor.send(Noop); - } - assert!( - result.is_ok(), - "stop route {route} failed to wake the mailbox" - ); - let reason = actor.wait_exit(); - if route == 3 { - assert!(matches!(reason, ExitReason::Panic(_))); - failing.unwrap().join(); - } else { - assert_eq!(reason, ExitReason::Normal); - } - } + crate::test_wait::bounded( + concat!(module_path!(), "::every_stop_route_wakes_an_idle_thread"), + move || { + for route in 0..4 { + let (stopped_tx, stopped) = mpsc::channel(); + let actor = Idle(stopped_tx).start(); + // A reply ensures startup is complete before the cancellation race. + actor.request(Noop).unwrap(); + let failing = if route == 3 { + let peer = Failing.start(); + actor.context().link(&peer.child_handle()); + Some(peer) + } else { + None + }; + match route { + 0 => actor.context().stop(), + 1 => actor.child_handle().stop(), + 2 => actor.cancellation_token.cancel(), + _ => failing.as_ref().unwrap().send(Crash).unwrap(), + } + if route == 3 { + // An untrapped link notice ends the idle actor without + // stopped() (decision 2): the wakeup is observed as its exit. + let reason = actor.child_handle().wait_exit_blocking(); + assert!(matches!(reason, ExitReason::Panic(_)), "got {reason:?}"); + assert!(stopped.try_recv().is_err(), "stopped() ran"); + failing.unwrap().join(); + continue; + } + let result = stopped.recv_timeout(Duration::from_secs(2)); + // A failed test must not hang cleanup on a broken wakeup callback. + if result.is_err() { + actor.child_handle().stop(); + let _ = actor.send(Noop); + } + assert!( + result.is_ok(), + "stop route {route} failed to wake the mailbox" + ); + let reason = actor.wait_exit(); + if route == 3 { + assert!(matches!(reason, ExitReason::Panic(_))); + failing.unwrap().join(); + } else { + assert_eq!(reason, ExitReason::Normal); + } + } + }, + ); } struct Busy { @@ -1900,47 +2948,63 @@ mod stop_wakeup_regression { type Result = (); } impl Handler for Busy { - fn handle(&mut self, _: Work, _: &Context) { + fn handle( + &mut self, + _: Work, + reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { self.handled.fetch_add(1, Ordering::SeqCst); let _ = self.entered.send(()); self.release.recv_timeout(Duration::from_secs(5)).unwrap(); + reply.reply(()); } } #[test] fn repeated_stop_during_a_handler_discards_queued_requests() { - let (entered_tx, entered) = mpsc::channel(); - let (release, release_rx) = mpsc::channel(); - let handled = Arc::new(AtomicUsize::new(0)); - let actor = Busy { - entered: entered_tx, - release: release_rx, - handled: handled.clone(), - } - .start(); - let first = actor.request_raw(Work).unwrap(); - entered.recv_timeout(Duration::from_secs(2)).unwrap(); - let replies: Vec<_> = (0..256).map(|_| actor.request_raw(Work).unwrap()).collect(); - std::thread::scope(|scope| { - for _ in 0..16 { - let child = actor.child_handle(); - scope.spawn(move || { - for _ in 0..256 { - child.stop(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::repeated_stop_during_a_handler_discards_queued_requests" + ), + move || { + let (entered_tx, entered) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let handled = Arc::new(AtomicUsize::new(0)); + let actor = Busy { + entered: entered_tx, + release: release_rx, + handled: handled.clone(), + } + .start(); + let first = actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap(); + entered.recv_timeout(Duration::from_secs(2)).unwrap(); + let replies: Vec<_> = (0..256) + .map(|_| actor.send_request(Work, DEFAULT_REQUEST_TIMEOUT).unwrap()) + .collect(); + std::thread::scope(|scope| { + for _ in 0..16 { + let child = actor.child_handle(); + scope.spawn(move || { + for _ in 0..256 { + child.stop(); + } + }); } }); - } - }); - release.send(()).unwrap(); - first.recv_timeout(Duration::from_secs(2)).unwrap(); - for reply in replies { - assert!(matches!( - reply.recv_timeout(Duration::from_secs(2)), - Err(RecvTimeoutError::Disconnected) - )); - } - assert_eq!(actor.wait_exit(), ExitReason::Normal); - assert_eq!(handled.load(Ordering::SeqCst), 1); + release.send(()).unwrap(); + first.recv().unwrap(); + for reply in replies { + assert!(matches!( + reply.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + } + assert_eq!(actor.wait_exit(), ExitReason::Normal); + assert_eq!(handled.load(Ordering::SeqCst), 1); + }, + ); } } @@ -1976,90 +3040,144 @@ mod api_coverage { } impl Actor for Echo {} impl Handler for Echo { - fn handle(&mut self, msg: Ask, _: &Context) -> u32 { - msg.0 + fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); } } impl Handler for Echo { - fn handle(&mut self, msg: StopAfter, ctx: &Context) { + fn handle( + &mut self, + msg: StopAfter, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { let _ = msg.0.recv_timeout(WAIT); ctx.stop(); + reply.reply(()); } } impl Handler for Echo { - fn handle(&mut self, _: OwnRecipient, ctx: &Context) -> (Recipient, ActorId) { - (ctx.recipient::(), ctx.id()) + fn handle( + &mut self, + _: OwnRecipient, + reply: crate::reply::PendingReply<'_, (Recipient, ActorId)>, + ctx: &Context, + ) { + reply.reply((ctx.recipient::(), ctx.id())); } } impl Handler for Echo { - fn handle(&mut self, _: Boom, _: &Context) { - panic!("boom"); + fn handle( + &mut self, + _: Boom, + _reply: crate::reply::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("boom"); + } } } #[test] fn context_recipient_and_identity_reach_their_actor() { - let actor = Echo.start(); - let (recipient, id) = actor.request(OwnRecipient).unwrap(); - assert_eq!(id, actor.id()); - assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); - recipient.send(Ask(1)).unwrap(); - actor.child_handle().stop(); - actor.join(); - assert!(matches!( - recipient.send(Ask(1)), - Err(ActorError::ActorStopped) - )); + crate::test_wait::bounded( + concat!( + module_path!(), + "::context_recipient_and_identity_reach_their_actor" + ), + move || { + let actor = Echo.start(); + let (recipient, id) = actor.request(OwnRecipient).unwrap(); + assert_eq!(id, actor.id()); + assert!(matches!(request(&*recipient, Ask(7), WAIT), Ok(7))); + recipient.send(Ask(1)).unwrap(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!(recipient.send(Ask(1)), Err(ActorError::NoProc))); + }, + ); } #[test] fn requests_to_a_stopped_actor_fail_without_waiting() { - let actor = Echo.start(); - let recipient = actor.recipient::(); - actor.child_handle().stop(); - actor.join(); - assert!(matches!( - actor.request_with_timeout(Ask(1), WAIT), - Err(ActorError::ActorStopped) - )); - assert!(matches!( - request(&*recipient, Ask(1), WAIT), - Err(ActorError::ActorStopped) - )); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_to_a_stopped_actor_fail_without_waiting" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + actor.child_handle().stop(); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4 of otp-exit-signals). + assert!(matches!( + actor.request_with_timeout(Ask(1), WAIT), + Err(ActorError::NoProc) + )); + assert!(matches!( + request(&*recipient, Ask(1), WAIT), + Err(ActorError::NoProc) + )); + }, + ); } #[test] fn admitted_requests_fail_when_the_handler_unwinds_before_replying() { - let actor = Echo.start(); - assert!(matches!( - actor.request_with_timeout(Boom, WAIT), - Err(ActorError::ActorStopped) - )); - let other = Echo.start(); - let recipient = other.recipient::(); - assert!(matches!( - request(&*recipient, Boom, WAIT), - Err(ActorError::ActorStopped) - )); + crate::test_wait::bounded( + concat!( + module_path!(), + "::admitted_requests_fail_when_the_handler_unwinds_before_replying" + ), + move || { + let actor = Echo.start(); + assert!(matches!( + actor.request_with_timeout(Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + let other = Echo.start(); + let recipient = other.recipient::(); + assert!(matches!( + request(&*recipient, Boom, WAIT), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + }, + ); } #[test] fn requests_time_out_while_the_actor_is_busy() { - let actor = Echo.start(); - let recipient = actor.recipient::(); - let (release, held) = std_mpsc::channel(); - actor.send(StopAfter(held)).unwrap(); - let short = Duration::from_millis(1); - assert!(matches!( - actor.request_with_timeout(Ask(1), short), - Err(ActorError::RequestTimeout) - )); - assert!(matches!( - request(&*recipient, Ask(1), short), - Err(ActorError::RequestTimeout) - )); - release.send(()).unwrap(); - actor.join(); + crate::test_wait::bounded( + concat!( + module_path!(), + "::requests_time_out_while_the_actor_is_busy" + ), + move || { + let actor = Echo.start(); + let recipient = actor.recipient::(); + let (release, held) = std_mpsc::channel(); + actor.send(StopAfter(held)).unwrap(); + let short = Duration::from_millis(1); + assert!(matches!( + actor.request_with_timeout(Ask(1), short), + Err(ActorError::RequestTimeout) + )); + assert!(matches!( + request(&*recipient, Ask(1), short), + Err(ActorError::RequestTimeout) + )); + release.send(()).unwrap(); + actor.join(); + }, + ); } struct Trapper { @@ -2071,93 +3189,136 @@ mod api_coverage { } } impl Handler for Trapper { - fn handle(&mut self, msg: Ask, _: &Context) -> u32 { - msg.0 + fn handle( + &mut self, + msg: Ask, + reply: crate::reply::PendingReply<'_, u32>, + _: &Context, + ) { + reply.reply(msg.0); } } impl Handler for Trapper { - fn handle(&mut self, msg: Unlink, ctx: &Context) { + fn handle( + &mut self, + msg: Unlink, + reply: crate::reply::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.unlink(&msg.0); + reply.reply(()); } } #[test] fn default_exit_received_keeps_a_trapping_actor_alive() { - let trapper = Trapper { trap: true }.start(); - assert!(matches!(trapper.request(Ask(0)), Ok(0))); - let peer = Echo.start(); - trapper.context().link(&peer.child_handle()); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - // The Exit is handled by the default `exit_received`, which ignores it. - assert!(matches!(trapper.request(Ask(3)), Ok(3))); - assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); - assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); + crate::test_wait::bounded( + concat!( + module_path!(), + "::default_exit_received_keeps_a_trapping_actor_alive" + ), + move || { + let trapper = Trapper { trap: true }.start(); + assert!(matches!(trapper.request(Ask(0)), Ok(0))); + let peer = Echo.start(); + trapper.context().link(&peer.child_handle()); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + // The Exit is handled by the default `exit_received`, which ignores it. + assert!(matches!(trapper.request(Ask(3)), Ok(3))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + assert_eq!(trapper.wait_exit(), ExitReason::Shutdown); + }, + ); } #[test] fn unlink_prevents_signal_delivery() { - // Not trapping: a linked peer's panic would take this actor down. - let trapper = Trapper { trap: false }.start(); - let peer = Echo.start(); - let peer_handle = peer.child_handle(); - trapper.context().link(&peer_handle); - trapper.request(Unlink(peer_handle.clone())).unwrap(); - // Unlinking yourself or an already-unlinked peer is a no-op. - trapper.request(Unlink(trapper.child_handle())).unwrap(); - trapper.request(Unlink(peer_handle)).unwrap(); - peer.send(Boom).unwrap(); - assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); - assert!(matches!(trapper.request(Ask(1)), Ok(1))); - assert!(trapper.exit_reason().is_none()); - trapper.child_handle().shutdown(); - trapper.join(); + crate::test_wait::bounded( + concat!(module_path!(), "::unlink_prevents_signal_delivery"), + move || { + // Not trapping: a linked peer's panic would take this actor down. + let trapper = Trapper { trap: false }.start(); + let peer = Echo.start(); + let peer_handle = peer.child_handle(); + trapper.context().link(&peer_handle); + trapper.request(Unlink(peer_handle.clone())).unwrap(); + // Unlinking yourself or an already-unlinked peer is a no-op. + trapper.request(Unlink(trapper.child_handle())).unwrap(); + trapper.request(Unlink(peer_handle)).unwrap(); + peer.send(Boom).unwrap(); + assert!(matches!(peer.wait_exit(), ExitReason::Panic(_))); + assert!(matches!(trapper.request(Ask(1)), Ok(1))); + assert!(trapper.exit_reason().is_none()); + trapper + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + trapper.join(); + }, + ); } #[test] fn opposite_links_between_the_same_pair_do_not_deadlock() { - // Each link/unlink locks both link tables. Both directions must take - // the locks in the same global order, or these two loops deadlock. - let a = Trapper { trap: true }.start(); - let b = Trapper { trap: true }.start(); - let (done, finished) = std_mpsc::channel(); - let workers: Vec<_> = [ - (a.context(), b.child_handle()), - (b.context(), a.child_handle()), - ] - .into_iter() - .map(|(own, peer)| { - let done = done.clone(); - std::thread::spawn(move || { - for _ in 0..2_000 { - own.link(&peer); - own.unlink(&peer); + crate::test_wait::bounded( + concat!( + module_path!(), + "::opposite_links_between_the_same_pair_do_not_deadlock" + ), + move || { + // Each link/unlink locks both link tables. Both directions must take + // the locks in the same global order, or these two loops deadlock. + let a = Trapper { trap: true }.start(); + let b = Trapper { trap: true }.start(); + let (done, finished) = std_mpsc::channel(); + let workers: Vec<_> = [ + (a.context(), b.child_handle()), + (b.context(), a.child_handle()), + ] + .into_iter() + .map(|(own, peer)| { + let done = done.clone(); + std::thread::spawn(move || { + for _ in 0..2_000 { + own.link(&peer); + own.unlink(&peer); + } + done.send(()).unwrap(); + }) + }) + .collect(); + for _ in 0..2 { + finished + .recv_timeout(WAIT) + .expect("opposite link directions deadlocked"); } - done.send(()).unwrap(); - }) - }) - .collect(); - for _ in 0..2 { - finished - .recv_timeout(WAIT) - .expect("opposite link directions deadlocked"); - } - for worker in workers { - worker.join().unwrap(); - } - for actor in [a, b] { - actor.child_handle().shutdown(); - actor.join(); - } + for worker in workers { + worker.join().unwrap(); + } + for actor in [a, b] { + actor + .child_handle() + .stop_with_reason(crate::ExitReason::Shutdown); + actor.join(); + } + }, + ); } #[test] fn debug_output_names_the_handle() { - let actor = Echo.start(); - assert!(format!("{actor:?}").starts_with("ActorRef")); - assert!(format!("{:?}", actor.context()).starts_with("Context")); - actor.child_handle().stop(); - actor.join(); + crate::test_wait::bounded( + concat!(module_path!(), "::debug_output_names_the_handle"), + move || { + let actor = Echo.start(); + assert!(format!("{actor:?}").starts_with("ActorRef")); + assert!(format!("{:?}", actor.context()).starts_with("Context")); + actor.child_handle().stop(); + actor.join(); + }, + ); } } diff --git a/concurrency/src/threads/jobs.rs b/concurrency/src/threads/jobs.rs new file mode 100644 index 000000000..7c34d47bd --- /dev/null +++ b/concurrency/src/threads/jobs.rs @@ -0,0 +1,754 @@ +//! `threads` mode pipe-to-self jobs (docs/design/deferred-replies.md ยง9): +//! one owned native thread per admitted job under `MAX_NATIVE_JOBS` +//! (`pipe_blocking`, `pipe_reply_blocking`), their envelopes and +//! `JobCtx`. The shared pool comes with rank 11b. +//! +//! A child module of `threads::actor`, so it reaches the private envelope and +//! context types. + +use super::{Actor, Context, Envelope, Handler, MailboxItem}; +use crate::message::Message; +use crate::reply::deadline::Expirable; +use crate::reply::jobs::{ + JobCredit, JobDeadline, JobError, JobGone, JobHandle, JobOptions, JobOutcome, JobState, + PipeError, PipeReplyError, SendScheduled, +}; +use crate::reply::slot::{Refused, Slot}; +use crate::reply::{now_ticks, PendingReply, ReplyTo}; +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, Mutex, Weak}; + +// --------------------------------------------------------------------------- +// Envelopes +// --------------------------------------------------------------------------- + +/// Returns one emit credit when the emitted message is dispatched, fenced or +/// dropped with the mailbox. +struct EmitCredit(Arc); +impl Drop for EmitCredit { + fn drop(&mut self) { + self.0.return_emit_credit(); + } +} + +/// A message emitted by a job; fenced at dispatch when the job is cancelled +/// or belongs to another instance. +struct EmitEnvelope { + msg: M, + credit: EmitCredit, +} + +impl Envelope for EmitEnvelope +where + A: Actor + Handler, + M: Message, +{ + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let EmitEnvelope { msg, credit } = *self; + let fenced = credit.0.fenced(ctx.id(), "emit"); + // The credit returns at dispatch (or here, when fenced). + drop(credit); + if fenced { + ctx.sender.failures().discard(msg); + return; + } + actor.handle(msg, PendingReply::detached(), ctx); + } +} + +type Done = Box) -> M + Send>; + +/// A job's terminal message: `done(outcome)` runs in the owner's turn and the +/// produced message is dispatched in the same turn. +struct TerminalEnvelope { + job: Arc, + done: Done, + outcome: JobOutcome, +} + +impl Envelope for TerminalEnvelope +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + fn handle(self: Box, actor: &mut A, ctx: &Context) { + let TerminalEnvelope { job, done, outcome } = *self; + let kind = if matches!(outcome, Err(JobError::DeadlineExceeded)) { + "deadline" + } else { + "done" + }; + if job.fenced(ctx.id(), kind) { + // Fenced: `done` and its captures (a linked token) drop in the + // turn, by the drop rules. + ctx.sender.failures().discard((done, outcome)); + return; + } + let msg = done(outcome); + actor.handle(msg, PendingReply::detached(), ctx); + } +} + +// --------------------------------------------------------------------------- +// Job runs +// --------------------------------------------------------------------------- + +/// The parts shared by a job's completion path and its deadline. +struct Run { + state: Arc, + ctx: Context, + done: Mutex>>, + /// The job's deadline (the heap holds a `Weak`): owned until the run is + /// dropped, so it still fires while a result waits for scheduled emits. + deadline: std::sync::OnceLock>, + /// What execution held (its credit, or its place in a linked slot), + /// handed over when execution ends and released when the terminal + /// message is enqueued (or the run is dropped): a result waiting for + /// scheduled emits keeps it, so `MAX_OUTSTANDING` still bounds it. + hold: Mutex>>, +} + +fn end_of(outcome: &JobOutcome) -> &'static str { + match outcome { + Ok(_) => "ok", + Err(JobError::Panicked(_)) => "panicked", + Err(JobError::DeadlineExceeded) => "deadline", + } +} + +impl Run +where + A: Actor + Handler, + M: Message, + T: Send + 'static, +{ + /// Take the terminal notification and enqueue `(done, outcome)`. + fn notify(&self, outcome: JobOutcome) { + let (_, unsent) = self + .state + .notify(end_of(&outcome), || self.enqueue_terminal(outcome)); + // A refused terminal message (`done` and the outcome) may drop on + // the deadline service thread: contained. + self.ctx.sender.failures().discard(unsent); + } + + fn enqueue_terminal(&self, outcome: JobOutcome) -> Option> { + let done = self.done.lock().unwrap_or_else(|p| p.into_inner()).take()?; + // Execution ended (or is still running past its deadline, and then + // keeps its hold): return the credit before the terminal message + // can be dispatched, so `done`'s turn can start the next job at the + // bound. Only atomics run here (a credit or the slot's job count). + let hold = self.hold.lock().unwrap_or_else(|p| p.into_inner()).take(); + drop(hold); + let envelope: Box + Send> = Box::new(TerminalEnvelope { + job: self.state.clone(), + done, + outcome, + }); + self.ctx + .sender + .send(MailboxItem::Message(envelope)) + .err() + .map(|e| e.0) + } + + /// The work's result: notify now, or, while delayed emits are still + /// scheduled, hold it for the last of them so the terminal message + /// stays last. `hold` is what execution held; the terminal message + /// releases it. + fn finish(self: &Arc, outcome: JobOutcome, hold: Box) { + *self.hold.lock().unwrap_or_else(|p| p.into_inner()) = Some(hold); + let end = end_of(&outcome); + let outcome = std::cell::Cell::new(Some(outcome)); + let unsent = self.state.finish( + end, + || self.enqueue_terminal(outcome.take()?), + || { + let run = self.clone(); + // Dropped unrun (a deadline or a cancel overtook it) possibly + // on the deadline service thread: contained. + let outcome = outcome.take().map(|outcome| { + crate::reply::jobs::Contained::new(outcome, self.ctx.sender.failures()) + }); + Box::new(move || { + if let Some(outcome) = outcome.and_then(|o| o.into_inner()) { + run.notify(outcome); + } + }) + }, + ); + self.ctx.sender.failures().discard(unsent); + } +} + +impl Drop for Run { + /// The last reference may go on the deadline service thread (a deadline + /// or a held result firing): `done` and its captures drop contained. + fn drop(&mut self) { + let done = self + .done + .get_mut() + .unwrap_or_else(|p| p.into_inner()) + .take(); + self.ctx.sender.failures().discard(done); + } +} + +/// Holds a `pipe_reply` job's place in its slot's `JOBS` count. +struct LinkedJob(Arc>); +impl Drop for LinkedJob { + fn drop(&mut self) { + self.0.job_end(); + } +} + +/// What a job holds while it executes; released when execution ends. +enum Hold { + Credit(#[allow(dead_code)] JobCredit), + Linked(#[allow(dead_code)] LinkedJob), +} + +/// Native slot (`MAX_NATIVE_JOBS`) held by a blocking job. +struct NativeSlot(Context); +impl Drop for NativeSlot { + fn drop(&mut self) { + self.0.jobs.release_native(); + } +} + +/// Marks the job ended in the owner's job set when execution ends. +struct Ended(Context, Arc); +impl Drop for Ended { + fn drop(&mut self) { + self.0.jobs.ended(&self.1); + } +} + +fn caught(result: std::thread::Result) -> JobOutcome { + result.map_err(|panic| { + let message = crate::error::panic_message(&*panic); + crate::mailbox::dispose_panic(panic); + JobError::Panicked(message) + }) +} + +/// The job's handle on its owner while it runs. +pub struct JobCtx { + state: Arc, + ctx: Context, +} + +impl std::fmt::Debug for JobCtx { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("JobCtx") + .field("cancelled", &self.state.is_cancelled()) + .finish_non_exhaustive() + } +} + +impl JobCtx { + /// Whether the job was cancelled (handle, key replacement, owner exit, + /// or opt-in caller give-up). Cancellation suppresses later results. + pub fn is_cancelled(&self) -> bool { + self.state.is_cancelled() + } + + /// Send `msg` to the owner as one of this job's messages, fenced like the + /// terminal one and never after it. Blocks while `emit_credits` messages + /// are in flight; fails with [`JobGone`] once the job is cancelled or its + /// terminal notification was taken. + pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + self.enqueue(msg) + } + + /// [`emit_blocking`](Self::emit_blocking) delivered at `at` instead of + /// now (OTP's `erlang:send_after` to the owner, fenced with the job). + /// Takes an emit credit now (blocking while all are in flight) and keeps + /// it until the message is dispatched. At `at` the message is enqueued + /// like an emit: dropped if the job was cancelled or replaced, or its + /// deadline passed. The job's terminal message waits for its scheduled + /// emits, so it stays last even when the work finishes first. + /// + /// `at` is an instant of the monotonic deadline clock. One already + /// reached (or in the past) is an immediate emit: the message is + /// enqueued at the call, ahead of later emits, so there is no need to + /// branch to [`emit_blocking`](Self::emit_blocking). + pub fn emit_at_blocking(&self, at: crate::Instant, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + self.state.acquire_blocking()?; + let credit = EmitCredit(self.state.clone()); + let ctx = self.ctx.clone(); + let send: SendScheduled = Box::new(move || { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + ctx.sender + .send(MailboxItem::Message(envelope)) + .map_err(|refused| Box::new(refused.0) as Box) + }); + JobState::schedule_emit( + &self.state, + spawned_rt::time::to_ticks(at), + send, + self.ctx.sender.failures(), + |at, item| self.ctx.replies.track(at, item), + ) + } + + fn enqueue(&self, msg: M) -> Result<(), JobGone> + where + A: Handler, + M: Message, + { + let credit = EmitCredit(self.state.clone()); + self.state + .enqueue_emit(|| { + let envelope: Box + Send> = Box::new(EmitEnvelope { msg, credit }); + self.ctx.sender.send(MailboxItem::Message(envelope)) + }) + .map_err(|(gone, unsent)| { + drop(unsent); + gone + }) + } +} + +// --------------------------------------------------------------------------- +// Context API +// --------------------------------------------------------------------------- + +/// The task a native worker thread runs once its job is spawned. +type NativeTask = Box; + +/// A native thread spawned at reservation and parked until its job is +/// spawned, so spawning from a permit cannot fail, with its native slot. +/// The slot stays here until the job is launched (it then moves into the +/// task): dropped unused, the worker returns the slot at once, on the +/// dropping thread, and the parked thread exits. +struct NativeWorker { + start: std::sync::mpsc::Sender, + native: NativeSlot, +} + +/// A blocking job's admission reserved ahead of building the work: +/// `ctx.reserve_blocking_job(opts)` checks the owner is running and takes a +/// `MAX_OUTSTANDING` credit, a native slot and the job's (parked) thread, so +/// a caller that is refused still holds what it would have moved into +/// `work` and `done`, and spawning from the permit cannot fail. Dropping an +/// unused permit returns everything. +#[must_use = "an unused permit only holds a credit and a native thread; spawn the job with it or drop it"] +pub struct BlockingJobPermit { + ctx: Context, + opts: JobOptions, + credit: JobCredit, + worker: NativeWorker, +} + +impl std::fmt::Debug for BlockingJobPermit { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("BlockingJobPermit").finish_non_exhaustive() + } +} + +impl BlockingJobPermit { + /// [`Context::pipe_blocking`] with this permit's options, credit and + /// thread; cannot fail. If the owner began exiting since the + /// reservation, nothing runs: `work` and `done` are dropped and the + /// handle reports the job cancelled (as owner exit cancels every job). + pub fn pipe_blocking(self, work: W, done: D) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + let BlockingJobPermit { + ctx, + opts, + credit, + worker, + } = self; + let state = JobState::new(ctx.id(), opts.emit_credits); + // Refused once the owner began exiting, in one step with the exit's + // close of admission (see `admit_job`). + if ctx.replies.is_exiting() || !ctx.admit_job(&state, &opts) { + drop((work, done, worker, credit)); + state.never_started(); + return JobHandle::new(state); + } + ctx.launch_blocking::( + opts, + state, + Hold::Credit(credit), + worker, + work, + Box::new(done), + ) + } +} + +impl Context { + /// Admit `state` into the owner's job set (the model's `Start`), after + /// every reversible reservation and before anything that cannot be + /// undone: refused once the owner's exit closed admission, in one step + /// with that close, so an admitted job is always cancelled (and joined) + /// by the owner's exit. + fn admit_job(&self, state: &Arc, opts: &JobOptions) -> bool { + self.jobs + .admit(state, opts.key.clone(), opts.deadline.is_some()) + } + + fn reserve_credit(&self) -> Result { + if self.replies.is_exiting() { + return Err(PipeError::OwnerStopping); + } + JobCredit::reserve(&self.replies).ok_or(PipeError::Overloaded) + } + + /// Reserve a native slot and spawn its thread, parked until the job is + /// spawned. A spawn failure releases the slot. + fn spawn_worker(&self) -> Result, PipeError> { + if !self.jobs.reserve_native() { + return Err(PipeError::NativeBudget); + } + let native = NativeSlot(self.clone()); + let (start, started) = std::sync::mpsc::channel::(); + std::thread::Builder::new() + .name("spawned-job".into()) + .spawn(move || { + if let Ok(task) = started.recv() { + task(); + } else { + // An unused permit was dropped: this thread only exits + // (the slot was already returned on the dropping thread). + fail_point!("job.native_worker_unused"); + } + }) + .map_err(PipeError::SpawnFailed)?; + Ok(NativeWorker { start, native }) + } + + /// Reserve a blocking job's admission (owner running, one credit, a + /// native slot and its thread) before building `work` and `done`: a + /// refusal leaves the caller everything it would have moved into them, + /// and spawning from the permit cannot fail. + /// + /// ```ignore + /// let Ok(permit) = ctx.reserve_blocking_job(JobOptions::new()) else { + /// return self.report(id, Verdict::Overloaded); // `id` was not moved + /// }; + /// permit.pipe_blocking(move |_| check(&object), move |res| Checked { id, res }); + /// ``` + pub fn reserve_blocking_job( + &self, + opts: JobOptions, + ) -> Result, PipeError> { + debug_assert!( + !opts.cancel_on_caller_gone, + "cancel_on_caller_gone applies to pipe_reply* only" + ); + let credit = self.reserve_credit()?; + let worker = self.spawn_worker()?; + Ok(BlockingJobPermit { + ctx: self.clone(), + opts, + credit, + worker, + }) + } + + fn arm_deadline(&self, opts: &JobOptions, run: &Arc>) + where + A: Handler, + M: Message, + T: Send + 'static, + { + let Some(at) = opts.deadline.map(spawned_rt::time::to_ticks) else { + return; + }; + let weak = Arc::downgrade(run); + let deadline = Arc::new(JobDeadline { + at, + state: run.state.clone(), + notify: Mutex::new(Some(Box::new(move || { + if let Some(run) = weak.upgrade() { + run.notify(Err(JobError::DeadlineExceeded)); + } + }))), + }); + // The run owns it; the heap only refers to it. + let _ = run.deadline.set(deadline.clone()); + if at <= now_ticks() { + deadline.expire(at); + } else { + let item: Weak = Arc::downgrade(&deadline) as Weak; + self.replies.track(at, item); + } + } + + fn launch_blocking( + &self, + opts: JobOptions, + state: Arc, + hold: Hold, + worker: NativeWorker, + work: W, + done: Done, + ) -> JobHandle + where + A: Handler, + M: Message, + T: Send + 'static, + R: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + { + let run = Arc::new(Run { + state: state.clone(), + ctx: self.clone(), + done: Mutex::new(Some(done)), + deadline: std::sync::OnceLock::new(), + hold: Mutex::new(None), + }); + self.arm_deadline(&opts, &run); + let ended = Ended(self.clone(), state.clone()); + let job = JobCtx { + state: state.clone(), + ctx: self.clone(), + }; + let NativeWorker { start, native } = worker; + let task: NativeTask = Box::new(move || { + let _ended = ended; + let result = std::panic::catch_unwind(AssertUnwindSafe(|| work(job))); + // Execution ended: release the native slot now; the run returns + // the credit when the terminal message is enqueued. + drop(native); + run.finish(caught(result), Box::new(hold)); + }); + // The worker thread is parked on `recv` until this send (or its + // sender's drop); it cannot have exited. + let _ = start.send(task); + JobHandle::new(state) + } + + /// Run blocking `work` as a job on an owned native thread, under the + /// actor's `MAX_NATIVE_JOBS` budget, and deliver `done(outcome)` as one + /// message in a later turn. A panic arrives as + /// `Err(JobError::Panicked)`. `work` gets the [`JobCtx`] to + /// check cancellation and `emit_blocking` fenced messages. Rejects at + /// once at a bound; never waits. On `Err`, `work` and `done` are + /// dropped: use [`reserve_blocking_job`](Self::reserve_blocking_job) to + /// keep them. + pub fn pipe_blocking( + &self, + opts: JobOptions, + work: W, + done: D, + ) -> Result + where + A: Handler, + M: Message, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome) -> M + Send + 'static, + { + Ok(self.reserve_blocking_job(opts)?.pipe_blocking(work, done)) + } + + /// Delegate a deferred reply: run blocking `work` on an owned native + /// thread, linked to `token`'s request, and deliver `done(outcome, token)` + /// in a later turn. The obligation and credit stay in this actor: the + /// token and its first linked job share one credit. On every error the + /// token comes back. With [`JobOptions::cancel_on_caller_gone`] the job + /// is cancelled when the caller leaves or the request's deadline passes + /// (off by default, as in OTP). + pub fn pipe_reply_blocking( + &self, + opts: JobOptions, + token: ReplyTo, + work: W, + done: D, + ) -> Result> + where + A: Handler, + M: Message, + R: Send + 'static, + T: Send + 'static, + W: FnOnce(JobCtx) -> T + Send + 'static, + D: FnOnce(JobOutcome, ReplyTo) -> M + Send + 'static, + { + let (state, slot, token) = self.link_reply(&opts, token)?; + let hold = Hold::Linked(LinkedJob(slot.clone())); + let worker = match self.spawn_worker() { + Ok(worker) => worker, + // Dropping the hold detaches the linked job from the slot. + Err(error) => { + drop(hold); + return Err(PipeReplyError::Failed { token, error }); + } + }; + if !self.admit_job(&state, &opts) { + drop((hold, worker)); + return Err(PipeReplyError::Failed { + token, + error: PipeError::OwnerStopping, + }); + } + if opts.cancel_on_caller_gone { + crate::reply::jobs::cancel_when_closed(&slot, &state); + } + #[cfg(feature = "lifecycle-trace")] + let work = { + let link = (self.id().raw(), Arc::as_ptr(&slot) as u64); + state.linked(link.1); + move |job| crate::lifecycle_trace::in_link(Some(link), || work(job)) + }; + let done: Done = Box::new(move |outcome| done(outcome, token)); + Ok(self.launch_blocking(opts, state, hold, worker, work, done)) + } + + /// Checks shared by the `pipe_reply*` calls: owner, owner exit, and the + /// attach CAS on the token's slot (which reserves a credit only when a + /// linked job already runs). + #[allow(clippy::type_complexity)] + fn link_reply( + &self, + opts: &JobOptions, + token: ReplyTo, + ) -> Result<(Arc, Arc>, ReplyTo), PipeReplyError> { + if token.owner() != self.id() { + return Err(PipeReplyError::WrongOwner(token)); + } + if self.replies.is_exiting() { + return Err(PipeReplyError::Failed { + token, + error: PipeError::OwnerStopping, + }); + } + let state = JobState::new(self.id(), opts.emit_credits); + let slot = token.slot_arc().clone(); + let attached = slot.attach(&now_ticks); + // The slot refused the job (reply traces: `PipeReply` handing the + // token back). + #[cfg(feature = "lifecycle-trace")] + if attached.is_err() { + crate::lifecycle_trace::reply( + self.id().raw(), + "PipeReply", + Arc::as_ptr(&slot) as u64, + "refused", + ); + } + match attached { + Ok(()) => Ok((state, slot, token)), + Err(Refused::Closed) => Err(PipeReplyError::Closed(token)), + Err(Refused::Overloaded) => Err(PipeReplyError::Failed { + token, + error: PipeError::Overloaded, + }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::threads::ActorStart; + + struct Owner; + impl Actor for Owner {} + struct Note; + impl Message for Note { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, _: Note, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review round 2 (agy 2): dropping an unused permit returns its + /// `MAX_NATIVE_JOBS` slot synchronously, so the next reservation + /// succeeds at once. The slot used to live on the parked thread and + /// return only when that thread woke up and exited. + #[test] + fn a_dropped_unused_permit_returns_its_native_slot_at_once() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::a_dropped_unused_permit_returns_its_native_slot_at_once" + ), + move || { + struct One; + impl Actor for One { + const MAX_NATIVE_JOBS: Option = Some(1); + } + let owner = One.start(); + let ctx = owner.context(); + for i in 0..2000 { + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .unwrap_or_else(|e| panic!("reservation {i} refused: {e}")); + drop(permit); + } + ctx.stop(); + owner.join(); + }, + ); + } + + /// Review P1 (Codex 13), threads mode: see the tasks-mode test. + #[test] + fn a_refused_emit_returns_its_credit_once() { + crate::test_wait::bounded( + concat!(module_path!(), "::a_refused_emit_returns_its_credit_once"), + move || { + let owner = Owner.start(); + for credits in [1, u16::MAX] { + let state = JobState::new(owner.id(), credits); + let job = JobCtx { + state: state.clone(), + ctx: owner.context(), + }; + state.acquire_blocking().unwrap(); + state.cancel(); + assert_eq!(job.enqueue(Note), Err(JobGone)); + assert_eq!(state.emit_credits(), credits, "credit returned twice"); + } + owner.context().stop(); + owner.join(); + }, + ); + } + + /// `cancel_on_caller_gone` applies to reply jobs only: a plain job has no + /// caller, so setting it there is a bug, caught in debug builds. + #[cfg(debug_assertions)] + #[test] + fn cancel_on_caller_gone_on_a_plain_job_is_a_bug() { + crate::test_wait::bounded( + concat!( + module_path!(), + "::cancel_on_caller_gone_on_a_plain_job_is_a_bug" + ), + move || { + let owner = Owner.start(); + let ctx = owner.context(); + let refused = std::panic::catch_unwind(AssertUnwindSafe(|| { + ctx.reserve_blocking_job(JobOptions::new().cancel_on_caller_gone()) + .map(drop) + })); + assert!(refused.is_err(), "the option was accepted on a plain job"); + let plain = ctx.reserve_blocking_job(JobOptions::new()).map(drop); + assert!(plain.is_ok()); + ctx.stop(); + owner.join(); + }, + ); + } +} diff --git a/concurrency/src/threads/mod.rs b/concurrency/src/threads/mod.rs index ee1e94b19..37f44979f 100644 --- a/concurrency/src/threads/mod.rs +++ b/concurrency/src/threads/mod.rs @@ -8,10 +8,19 @@ mod timer_tests; #[cfg(test)] mod stream_tests; +pub use crate::reply::jobs::{ + JobError, JobGone, JobHandle, JobKey, JobOptions, JobOutcome, PipeError, PipeReplyError, +}; +pub use crate::reply::{ + Deadline, DeferError, Delivery, InsertError, InsertErrorKind, PendingReplies, PendingReply, + ReplyReceiver, ReplyTo, +}; pub use crate::response::Response; +pub use crate::Instant; pub use actor::{ request, send_message_on, Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, DEFAULT_REQUEST_TIMEOUT, }; +pub use actor::{BlockingJobPermit, JobCtx, DEFAULT_STOP_DEADLINE}; pub use stream::spawn_listener; pub use time::{send_after, send_interval, TimerHandle}; diff --git a/concurrency/src/threads/stream_tests.rs b/concurrency/src/threads/stream_tests.rs index f5cb01db8..e16004936 100644 --- a/concurrency/src/threads/stream_tests.rs +++ b/concurrency/src/threads/stream_tests.rs @@ -28,99 +28,125 @@ impl Message for GetItems { impl Actor for Collector {} impl Handler for Collector { - fn handle(&mut self, msg: Push, _ctx: &Context) { + fn handle( + &mut self, + msg: Push, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.items.push(msg.value); self.done.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } impl Handler for Collector { - fn handle(&mut self, _msg: GetItems, _ctx: &Context) -> Vec { - self.items.clone() + fn handle( + &mut self, + _msg: GetItems, + reply: crate::reply::PendingReply<'_, Vec>, + _ctx: &Context, + ) { + reply.reply(self.items.clone()); } } #[test] fn listener_consumes_iterator() { - let done = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: done.clone(), - } - .start(); - - let ctx = Context::from_ref(&actor); - let items = vec![ - Push { value: 1 }, - Push { value: 2 }, - Push { value: 3 }, - Push { value: 4 }, - Push { value: 5 }, - ]; - let handle = spawn_listener(ctx, items); - handle.join().unwrap(); - // Give the actor time to process all messages - while done.load(Ordering::SeqCst) < 5 { - rt::sleep(Duration::from_millis(10)); - } - - let result = actor.request(GetItems).unwrap(); - assert_eq!(result, vec![1, 2, 3, 4, 5]); + crate::test_wait::bounded( + concat!(module_path!(), "::listener_consumes_iterator"), + move || { + let done = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: done.clone(), + } + .start(); + + let ctx = actor.context(); + let items = vec![ + Push { value: 1 }, + Push { value: 2 }, + Push { value: 3 }, + Push { value: 4 }, + Push { value: 5 }, + ]; + let handle = spawn_listener(ctx, items); + handle.join().unwrap(); + // Give the actor time to process all messages + while done.load(Ordering::SeqCst) < 5 { + rt::sleep(Duration::from_millis(10)); + } + + let result = actor.request(GetItems).unwrap(); + assert_eq!(result, vec![1, 2, 3, 4, 5]); + }, + ); } #[test] fn listener_stops_on_cancellation() { - let done = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: done.clone(), - } - .start(); - - let ctx = Context::from_ref(&actor); - - // An iterator that blocks between items, giving us time to cancel - let iter = (1..=100).map(move |i| { - if i > 1 { - rt::sleep(Duration::from_millis(50)); - } - Push { value: i } - }); - - let _handle = spawn_listener(ctx, iter); - - // Let at least one item through, then stop the actor - let first = wait_for(|| done.load(Ordering::SeqCst), |n| *n > 0); - assert!(first > 0, "listener never delivered an item"); - actor.context().stop(); - actor.join(); - - let processed = done.load(Ordering::SeqCst); - assert!( - processed < 100, - "Expected partial processing, but all 100 items were processed" - ); - assert!( - processed > 0, - "Expected at least some items to be processed" + crate::test_wait::bounded( + concat!(module_path!(), "::listener_stops_on_cancellation"), + move || { + let done = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: done.clone(), + } + .start(); + + let ctx = actor.context(); + + // An iterator that blocks between items, giving us time to cancel + let iter = (1..=100).map(move |i| { + if i > 1 { + rt::sleep(Duration::from_millis(50)); + } + Push { value: i } + }); + + let _handle = spawn_listener(ctx, iter); + + // Let at least one item through, then stop the actor + let first = wait_for(|| done.load(Ordering::SeqCst), |n| *n > 0); + assert!(first > 0, "listener never delivered an item"); + actor.context().stop(); + actor.join(); + + let processed = done.load(Ordering::SeqCst); + assert!( + processed < 100, + "Expected partial processing, but all 100 items were processed" + ); + assert!( + processed > 0, + "Expected at least some items to be processed" + ); + }, ); } #[test] fn stopped_actor_does_not_poll_iterator() { - let polls = Arc::new(AtomicU64::new(0)); - let actor = Collector { - items: vec![], - done: Default::default(), - } - .start(); - actor.child_handle().stop(); - actor.join(); - let count = polls.clone(); - let iter = std::iter::from_fn(move || { - count.fetch_add(1, Ordering::SeqCst); - None:: - }); - spawn_listener(actor.context(), iter).join().unwrap(); - assert_eq!(polls.load(Ordering::SeqCst), 0); + crate::test_wait::bounded( + concat!(module_path!(), "::stopped_actor_does_not_poll_iterator"), + move || { + let polls = Arc::new(AtomicU64::new(0)); + let actor = Collector { + items: vec![], + done: Default::default(), + } + .start(); + actor.child_handle().stop(); + actor.join(); + let count = polls.clone(); + let iter = std::iter::from_fn(move || { + count.fetch_add(1, Ordering::SeqCst); + None:: + }); + spawn_listener(actor.context(), iter).join().unwrap(); + assert_eq!(polls.load(Ordering::SeqCst), 0); + }, + ); } diff --git a/concurrency/src/threads/timer_tests.rs b/concurrency/src/threads/timer_tests.rs index 004a8bf4b..62d4f45e1 100644 --- a/concurrency/src/threads/timer_tests.rs +++ b/concurrency/src/threads/timer_tests.rs @@ -46,47 +46,69 @@ impl Actor for Repeater { } impl Handler for Repeater { - fn handle(&mut self, _msg: Inc, _ctx: &Context) { + fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.count += 1; + reply.reply(()); } } impl Handler for Repeater { - fn handle(&mut self, _msg: StopTimer, _ctx: &Context) { + fn handle( + &mut self, + _msg: StopTimer, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { if let Some(ct) = self.cancellation_token.clone() { ct.cancel(); } + reply.reply(()); } } impl Handler for Repeater { - fn handle(&mut self, _msg: GetRepCount, _ctx: &Context) -> i32 { - self.count + fn handle( + &mut self, + _msg: GetRepCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + reply.reply(self.count); } } #[test] pub fn test_send_interval_and_cancellation() { - let repeater = Repeater::new(0).start(); - - // The interval keeps firing. - let count = wait_for( - || repeater.request(GetRepCount), - |count| matches!(count, Ok(n) if *n >= 3), - ) - .unwrap(); - assert!(count >= 3, "interval stopped firing at {count}"); - - // StopTimer is handled before the next request. The timer thread checks - // the token and then sends, so one tick that raced the cancellation may - // still land, but no more than one. - repeater.send(StopTimer).unwrap(); - let stopped_at = repeater.request(GetRepCount).unwrap(); - rt::sleep(Duration::from_millis(500)); - let after = repeater.request(GetRepCount).unwrap(); - assert!( - after <= stopped_at + 1, - "interval kept firing after cancellation: {stopped_at} -> {after}" + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_interval_and_cancellation"), + move || { + let repeater = Repeater::new(0).start(); + + // The interval keeps firing. + let count = wait_for( + || repeater.request(GetRepCount), + |count| matches!(count, Ok(n) if *n >= 3), + ) + .unwrap(); + assert!(count >= 3, "interval stopped firing at {count}"); + + // StopTimer is handled before the next request. The timer thread checks + // the token and then sends, so one tick that raced the cancellation may + // still land, but no more than one. + repeater.send(StopTimer).unwrap(); + let stopped_at = repeater.request(GetRepCount).unwrap(); + rt::sleep(Duration::from_millis(500)); + let after = repeater.request(GetRepCount).unwrap(); + assert!( + after <= stopped_at + 1, + "interval kept firing after cancellation: {stopped_at} -> {after}" + ); + }, ); } @@ -119,71 +141,100 @@ impl Delayed { impl Actor for Delayed {} impl Handler for Delayed { - fn handle(&mut self, _msg: Inc, _ctx: &Context) { + fn handle( + &mut self, + _msg: Inc, + reply: crate::reply::PendingReply<'_, ()>, + _ctx: &Context, + ) { self.count += 1; + reply.reply(()); } } impl Handler for Delayed { - fn handle(&mut self, _msg: GetDelCount, _ctx: &Context) -> i32 { - self.count + fn handle( + &mut self, + _msg: GetDelCount, + reply: crate::reply::PendingReply<'_, i32>, + _ctx: &Context, + ) { + reply.reply(self.count); } } impl Handler for Delayed { - fn handle(&mut self, _msg: StopDelayed, ctx: &Context) -> i32 { - ctx.stop(); - self.count + fn handle( + &mut self, + _msg: StopDelayed, + reply: crate::reply::PendingReply<'_, i32>, + ctx: &Context, + ) { + let __reply_value: i32 = { + ctx.stop(); + self.count + }; + reply.reply(__reply_value); } } #[test] pub fn test_send_after_and_cancellation() { - let actor = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_and_cancellation"), + move || { + let actor = Delayed::new(0).start(); - let ctx = Context::from_ref(&actor); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for( - || actor.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .unwrap(); - assert_eq!(1, count); + let count = wait_for( + || actor.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .unwrap(); + assert_eq!(1, count); - let ctx = Context::from_ref(&actor); - let timer = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let timer = send_after(Duration::from_millis(100), ctx, Inc); - timer.cancellation_token.cancel(); + timer.cancellation_token.cancel(); - rt::sleep(Duration::from_millis(200)); + rt::sleep(Duration::from_millis(200)); - let count2 = actor.request(GetDelCount).unwrap(); - assert_eq!(1, count2); + let count2 = actor.request(GetDelCount).unwrap(); + assert_eq!(1, count2); + }, + ); } #[test] pub fn test_send_after_actor_shutdown() { - let actor = Delayed::new(0).start(); + crate::test_wait::bounded( + concat!(module_path!(), "::test_send_after_actor_shutdown"), + move || { + let actor = Delayed::new(0).start(); - let ctx = Context::from_ref(&actor); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count = wait_for( - || actor.request(GetDelCount), - |count| matches!(count, Ok(n) if *n >= 1), - ) - .unwrap(); - assert_eq!(1, count); + let count = wait_for( + || actor.request(GetDelCount), + |count| matches!(count, Ok(n) if *n >= 1), + ) + .unwrap(); + assert_eq!(1, count); - let ctx = Context::from_ref(&actor); - let _ = send_after(Duration::from_millis(100), ctx, Inc); + let ctx = actor.context(); + let _ = send_after(Duration::from_millis(100), ctx, Inc); - let count2 = actor.request(StopDelayed).unwrap(); + let count2 = actor.request(StopDelayed).unwrap(); - rt::sleep(Duration::from_millis(200)); + rt::sleep(Duration::from_millis(200)); - assert_eq!(1, count2); + assert_eq!(1, count2); + }, + ); } // --- T4: threads timers use the monotonic clock (`recv_timeout`) --- @@ -202,8 +253,11 @@ impl Message for Fired { type Result = (); } impl Handler for Clocked { - fn handle(&mut self, _: Fired, _: &Context) { - self.handled.lock().unwrap().push(std::time::Instant::now()); + fn handle(&mut self, _: Fired, __reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let __value: () = { + self.handled.lock().unwrap().push(std::time::Instant::now()); + }; + __reply.reply(__value); } } @@ -219,12 +273,15 @@ impl Message for Stamp { type Result = (); } impl Handler for Clocked { - fn handle(&mut self, _: Stamp, _: &Context) { - self.handled.lock().unwrap().push(std::time::Instant::now()); - // The first tick stalls the actor for several periods. - if let Some(stall) = self.stall.take() { - rt::sleep(stall); - } + fn handle(&mut self, _: Stamp, __reply: crate::reply::PendingReply<'_, ()>, _: &Context) { + let __value: () = { + self.handled.lock().unwrap().push(std::time::Instant::now()); + // The first tick stalls the actor for several periods. + if let Some(stall) = self.stall.take() { + rt::sleep(stall); + } + }; + __reply.reply(__value); } } @@ -232,45 +289,53 @@ impl Handler for Clocked { /// fires early, fires once, and a long delay cancelled halfway never fires. #[test] pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { - let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let actor = Clocked { - handled: handled.clone(), - stall: None, - } - .start(); - let delay = Duration::from_millis(300); - let long = Duration::from_millis(1_500); - let armed = std::time::Instant::now(); - let _short = send_after(delay, Context::from_ref(&actor), Fired); - let cancelled = send_after(long, Context::from_ref(&actor), Fired); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 1) >= 1, - "timer never fired" - ); - let fired = handled.lock().unwrap()[0]; - assert!( - fired.duration_since(armed) >= delay, - "T4: send_after fired after {:?}, before its {delay:?}", - fired.duration_since(armed) + crate::test_wait::bounded( + concat!( + module_path!(), + "::threads_send_after_uses_the_monotonic_clock_over_long_delays" + ), + move || { + let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let actor = Clocked { + handled: handled.clone(), + stall: None, + } + .start(); + let delay = Duration::from_millis(300); + let long = Duration::from_millis(1_500); + let armed = std::time::Instant::now(); + let _short = send_after(delay, actor.context(), Fired); + let cancelled = send_after(long, actor.context(), Fired); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 1) >= 1, + "timer never fired" + ); + let fired = handled.lock().unwrap()[0]; + assert!( + fired.duration_since(armed) >= delay, + "T4: send_after fired after {:?}, before its {delay:?}", + fired.duration_since(armed) + ); + cancelled.cancellation_token.cancel(); + let _uncancelled = send_after(long, actor.context(), Fired); + let armed_long = std::time::Instant::now(); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 2) >= 2, + "timer never fired" + ); + let fired_long = handled.lock().unwrap()[1]; + assert!( + fired_long.duration_since(armed_long) >= long, + "T4: long send_after fired after {:?}", + fired_long.duration_since(armed_long) + ); + // The cancelled 1.5 s timer was due before the uncancelled one fired. + assert!(armed + long <= fired_long); + assert_eq!(handled.lock().unwrap().len(), 2, "T1: fired more than once"); + actor.child_handle().stop(); + actor.join(); + }, ); - cancelled.cancellation_token.cancel(); - let _uncancelled = send_after(long, Context::from_ref(&actor), Fired); - let armed_long = std::time::Instant::now(); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 2) >= 2, - "timer never fired" - ); - let fired_long = handled.lock().unwrap()[1]; - assert!( - fired_long.duration_since(armed_long) >= long, - "T4: long send_after fired after {:?}", - fired_long.duration_since(armed_long) - ); - // The cancelled 1.5 s timer was due before the uncancelled one fired. - assert!(armed + long <= fired_long); - assert_eq!(handled.lock().unwrap().len(), 2, "T1: fired more than once"); - actor.child_handle().stop(); - actor.join(); } /// T2/T4: threads `send_interval` is fixed delay on the monotonic clock: @@ -278,29 +343,37 @@ pub fn threads_send_after_uses_the_monotonic_clock_over_long_delays() { /// stalled for 3.5 periods, so there is no burst to catch up. #[test] pub fn threads_send_interval_is_fixed_delay_while_the_actor_stalls() { - let period = Duration::from_millis(50); - let sends = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); - let actor = Clocked { - handled: handled.clone(), - stall: Some(period * 7 / 2), - } - .start(); - let timer = send_interval(period, Context::from_ref(&actor), Stamp(sends.clone())); - assert!( - wait_for(|| handled.lock().unwrap().len(), |n| *n >= 10) >= 10, - "timer never fired" + crate::test_wait::bounded( + concat!( + module_path!(), + "::threads_send_interval_is_fixed_delay_while_the_actor_stalls" + ), + move || { + let period = Duration::from_millis(50); + let sends = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let handled = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let actor = Clocked { + handled: handled.clone(), + stall: Some(period * 7 / 2), + } + .start(); + let timer = send_interval(period, actor.context(), Stamp(sends.clone())); + assert!( + wait_for(|| handled.lock().unwrap().len(), |n| *n >= 10) >= 10, + "timer never fired" + ); + timer.cancellation_token.cancel(); + let sends = sends.lock().unwrap().clone(); + assert!(sends.len() >= 10, "{}", sends.len()); + for pair in sends.windows(2) { + assert!( + pair[1].duration_since(pair[0]) >= period, + "T2/T4: two sends {:?} apart, period {period:?}", + pair[1].duration_since(pair[0]) + ); + } + actor.child_handle().stop(); + actor.join(); + }, ); - timer.cancellation_token.cancel(); - let sends = sends.lock().unwrap().clone(); - assert!(sends.len() >= 10, "{}", sends.len()); - for pair in sends.windows(2) { - assert!( - pair[1].duration_since(pair[0]) >= period, - "T2/T4: two sends {:?} apart, period {period:?}", - pair[1].duration_since(pair[0]) - ); - } - actor.child_handle().stop(); - actor.join(); } diff --git a/concurrency/tests/actor_scenarios.proptest-regressions b/concurrency/tests/actor_scenarios.proptest-regressions new file mode 100644 index 000000000..4b880b83b --- /dev/null +++ b/concurrency/tests/actor_scenarios.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc ff1196c6c1795d0ebaecb9a90df2fc2bae945dc1b58db213e805227cbc55510c # shrinks to count = 4, operations = [(11, 21)], endings = [0, 2, 1, 2, 1] diff --git a/concurrency/tests/actor_scenarios.rs b/concurrency/tests/actor_scenarios.rs index 5d2bd22a1..d88414da5 100644 --- a/concurrency/tests/actor_scenarios.rs +++ b/concurrency/tests/actor_scenarios.rs @@ -37,8 +37,13 @@ mod tasks { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - async fn handle(&mut self, command: Command, ctx: &Context) -> Reply { - match command { + async fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::PendingReply<'_, Reply>, + ctx: &Context, + ) { + reply.reply(match command { Command::Work(id) => { self.0 .processed @@ -53,12 +58,18 @@ mod tasks { Reply::Unwatched } Command::Boom => panic!("scenario injected panic"), - } + }); } } impl Handler for TestActor { - async fn handle(&mut self, event: Down, _: &Context) { + async fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.events.send(event); + reply.reply(()); } } } @@ -69,8 +80,13 @@ mod threads { pub struct TestActor(pub Worker); impl Actor for TestActor {} impl Handler for TestActor { - fn handle(&mut self, command: Command, ctx: &Context) -> Reply { - match command { + fn handle( + &mut self, + command: Command, + reply: spawned_concurrency::PendingReply<'_, Reply>, + ctx: &Context, + ) { + reply.reply(match command { Command::Work(id) => { self.0 .processed @@ -85,12 +101,18 @@ mod threads { Reply::Unwatched } Command::Boom => panic!("scenario injected panic"), - } + }); } } impl Handler for TestActor { - fn handle(&mut self, event: Down, _: &Context) { + fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.events.send(event); + reply.reply(()); } } } @@ -142,8 +164,14 @@ impl Handle { fn abandon(&self, cmd: Command) { match self { #[cfg(feature = "async")] - Self::Tasks(a) => drop(a.request_raw(cmd).expect("live actor")), - Self::Threads(a) => drop(a.request_raw(cmd).expect("live actor")), + Self::Tasks(a) => drop( + a.send_request(cmd, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("live actor"), + ), + Self::Threads(a) => drop( + a.send_request(cmd, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("live actor"), + ), } } } @@ -242,7 +270,8 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) let child = actor.child(); let ending = endings[i % endings.len()] % 3; // Compete monitor registration and a request with termination. A - // monitor must observe termination even if registration loses the race. + // monitor must observe termination even if registration loses the + // race: then its single Down carries NoProc (E4). let gate = Arc::new(Barrier::new(2)); let racer = actor.clone(); let gate2 = gate.clone(); @@ -280,7 +309,9 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) assert_eq!(reply, Reply::Work(race_id)); assert!(logs[i].lock().unwrap().contains(&race_id)); } - Err(ActorError::ActorStopped) => {} + Err(ActorError::ActorStopped(_)) => {} + // Sent after the death point (v0.8, decision 4). + Err(ActorError::NoProc) => {} Err(other) => panic!("request failed to resolve during shutdown: {other:?}"), } let actual = logs[i].lock().unwrap(); @@ -290,7 +321,7 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) ); assert!(matches!( actor.request(&rt, Command::Work(u64::MAX)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); expected_down.push((monitor, reason)); } @@ -301,7 +332,11 @@ fn scenario(backend: u8, count: usize, operations: &[(u8, u8)], endings: &[u8]) .position(|(id, _)| *id == event.monitor_ref) .expect("duplicate or unknown monitor"); let (_, reason) = expected_down.remove(index); - assert_eq!(event.reason, reason); + assert!( + event.reason == reason || event.reason == ExitReason::NoProc, + "Down {:?}, final reason {reason:?}", + event.reason + ); } assert!(expected_down.is_empty()); observer.child().stop(); diff --git a/concurrency/tests/backend_experiments.rs b/concurrency/tests/backend_experiments.rs index 1da83eef9..7a231c8e2 100644 --- a/concurrency/tests/backend_experiments.rs +++ b/concurrency/tests/backend_experiments.rs @@ -20,22 +20,38 @@ impl Message for Nested { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Probe, _: &Context) -> u64 { - let child = tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 - }); - child.await.unwrap() + async fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + let child = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }); + child.await.unwrap() + }; + reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: Nested, _: &Context) -> bool { - let (tx, rx) = mpsc::channel(); - tokio::spawn(async move { - let _ = tx.send(()); - }); - // Characterizes progress of a detached helper while its actor blocks. - rx.recv_timeout(Duration::from_millis(100)).is_ok() + async fn handle( + &mut self, + _: Nested, + reply: spawned_concurrency::PendingReply<'_, bool>, + _: &Context, + ) { + let __reply_value: bool = { + let (tx, rx) = mpsc::channel(); + tokio::spawn(async move { + let _ = tx.send(()); + }); + // Characterizes progress of a detached helper while its actor blocks. + rx.recv_timeout(Duration::from_millis(100)).is_ok() + }; + reply.reply(__reply_value); } } #[test] @@ -66,26 +82,34 @@ impl Message for Network { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Network, _: &Context) -> u64 { - use std::io::{Read, Write}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - let server = std::thread::spawn(move || { - let (mut socket, _) = listener.accept().unwrap(); - socket - .set_read_timeout(Some(Duration::from_secs(3))) - .unwrap(); + async fn handle( + &mut self, + _: Network, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + use std::io::{Read, Write}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = std::thread::spawn(move || { + let (mut socket, _) = listener.accept().unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + let mut data = [0; 8]; + socket.read_exact(&mut data).unwrap(); + socket.write_all(&data).unwrap(); + }); + let mut socket = tokio::net::TcpStream::connect(addr).await.unwrap(); + socket.write_all(&42u64.to_le_bytes()).await.unwrap(); let mut data = [0; 8]; - socket.read_exact(&mut data).unwrap(); - socket.write_all(&data).unwrap(); - }); - let mut socket = tokio::net::TcpStream::connect(addr).await.unwrap(); - socket.write_all(&42u64.to_le_bytes()).await.unwrap(); - let mut data = [0; 8]; - socket.read_exact(&mut data).await.unwrap(); - server.join().unwrap(); - u64::from_le_bytes(data) + socket.read_exact(&mut data).await.unwrap(); + server.join().unwrap(); + u64::from_le_bytes(data) + }; + reply.reply(__reply_value); } } #[test] diff --git a/concurrency/tests/backend_resource_limits.rs b/concurrency/tests/backend_resource_limits.rs index 73690a584..ff396ee03 100644 --- a/concurrency/tests/backend_resource_limits.rs +++ b/concurrency/tests/backend_resource_limits.rs @@ -33,7 +33,14 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } // Not simulated: Blocking backend workers are OS threads. #[cfg(not(spawned_sim))] @@ -181,21 +188,21 @@ fn descriptor_limit_child() { tokio::time::timeout(LIMIT, actor.wait_exit()) .await .unwrap(), - spawned_concurrency::ExitReason::Kill + spawned_concurrency::ExitReason::Killed ); assert_eq!( actor.exit_reason(), - Some(spawned_concurrency::ExitReason::Kill) + Some(spawned_concurrency::ExitReason::Killed) ); let child = actor.child_handle(); assert!(!child.is_alive()); assert_eq!( child.exit_reason(), - Some(spawned_concurrency::ExitReason::Kill) + Some(spawned_concurrency::ExitReason::Killed) ); assert_eq!( child.wait_exit_blocking(), - spawned_concurrency::ExitReason::Kill + spawned_concurrency::ExitReason::Killed ); } } diff --git a/concurrency/tests/borrowed_runtime.rs b/concurrency/tests/borrowed_runtime.rs index 14bed951a..278a61877 100644 --- a/concurrency/tests/borrowed_runtime.rs +++ b/concurrency/tests/borrowed_runtime.rs @@ -20,13 +20,23 @@ impl Message for Ambient { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Identity, _: &Context) -> ThreadId { - std::thread::current().id() + async fn handle( + &mut self, + _: Identity, + reply: spawned_concurrency::PendingReply<'_, ThreadId>, + _: &Context, + ) { + reply.reply(std::thread::current().id()); } } impl Handler for Worker { - async fn handle(&mut self, _: Ambient, _: &Context) -> ActorRef { - Worker.start() + async fn handle( + &mut self, + _: Ambient, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, + _: &Context, + ) { + reply.reply(Worker.start()); } } diff --git a/concurrency/tests/canary_feedback.rs b/concurrency/tests/canary_feedback.rs index 91aeb89c2..8b8ba7902 100644 --- a/concurrency/tests/canary_feedback.rs +++ b/concurrency/tests/canary_feedback.rs @@ -36,14 +36,26 @@ impl Message for Quit { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, hold: Hold, _: &Context) { + async fn handle( + &mut self, + hold: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = hold.entered.send(()); let _ = hold.release.await; + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, quit: Quit, ctx: &Context) { + async fn handle( + &mut self, + quit: Quit, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop_with_reason(quit.0); + reply.reply(()); } } @@ -91,14 +103,14 @@ async fn unfinished_report_identifies_child_and_active_turn() { assert_eq!(detail.message_type, None); assert_eq!(detail.turn_age, None); } + // The system kills what remains at grace (E3); let the kill land + // before the turn returns, so the outcome does not race it. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); let report = report.wait(LIMIT).await; assert!(report.complete); assert!(report.unfinished_actor_details.is_empty()); - assert_eq!( - actor.child_handle().exit_reason(), - Some(ExitReason::Shutdown) - ); + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } // Not simulated: uses spawn_blocking. @@ -192,16 +204,14 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { let system = ActorSystem::new().unwrap(); let group = system.async_group("canary-tracing", 1).unwrap(); let mut builder = system - .supervisor(SupervisorConfig { - max_restarts: 1, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(1).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = builder .add( @@ -243,21 +253,22 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { // Not simulated: the cohort case blocks a spawn_blocking helper. #[cfg(not(spawned_sim))] { - // The healthy collateral child takes the last retry grant. Its failing - // sibling must give up before a 60-second backoff, and lookups must report - // owner failure even while the healthy sibling's owned cleanup is pending. + // The round is the restart action: with no grant left the supervisor + // gives up when it would select the round, before the origin's + // 60-second backoff, and lookups must report owner failure even while + // the healthy sibling's owned cleanup is pending. let mut cohort = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 1, - backoff: Backoff { - initial: Duration::from_secs(60), - maximum: Duration::from_secs(60), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(0) + .backoff( + Backoff::default() + .initial(Duration::from_secs(60)) + .maximum(Duration::from_secs(60)) + .jitter(false), + ), + ) .unwrap(); let (release, gate) = std::sync::mpsc::channel(); let gate = Arc::new(Mutex::new(Some(gate))); @@ -266,7 +277,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { ChildSpec::new("canary-healthy", &group, move |ctx| { let gate = gate.clone(); async move { - if ctx.generation() == 2 { + if ctx.generation() == 1 { let gate = gate.lock().unwrap().take().unwrap(); let (entered, arrival) = oneshot::channel(); ctx.spawn_blocking_helper(move || { @@ -308,7 +319,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { healthy.current(), Err(ChildUnavailable(ChildState::Failed)) )); - assert_eq!(healthy.snapshot().generation, 2); + assert_eq!(healthy.snapshot().generation, 1); release.send(()).unwrap(); spawned_rt::tasks::timeout(LIMIT, cohort.join()) .await @@ -337,7 +348,7 @@ async fn supervision_logs_restart_budget_exhaustion_and_give_up() { && fields.get("child").map(String::as_str) == Some("canary-give-up") && fields .get("error") - .is_some_and(|e| e.contains("non-restartable child")))); + .is_some_and(|e| e.contains("failed to start child")))); let exhausted: Vec<_> = events .iter() .filter(|(_, fields)| { @@ -397,7 +408,12 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { } } impl Handler for Panicking { - async fn handle(&mut self, _: Quit, _: &Context) { + async fn handle( + &mut self, + _: Quit, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("diagnostic turn cleanup regression"); } } @@ -425,12 +441,14 @@ async fn panicked_turn_is_cleared_before_stopped_hook() { .unwrap(); assert_eq!(detail.message_type, None); assert_eq!(detail.turn_age, None); + // Let the system's kill at grace land before stopped() returns: the + // report can be ready just before the kill is sent. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); - assert!(matches!( - actor.child_handle().exit_reason(), - Some(ExitReason::Panic(_)) - )); + // The system's kill at grace reached it inside stopped(), before its + // death point: Killed replaces the panic (E1). + assert_eq!(actor.child_handle().exit_reason(), Some(ExitReason::Killed)); } #[cfg(feature = "diagnostics")] diff --git a/concurrency/tests/cancellation_safety.rs b/concurrency/tests/cancellation_safety.rs index fb37f5199..d82d99918 100644 --- a/concurrency/tests/cancellation_safety.rs +++ b/concurrency/tests/cancellation_safety.rs @@ -161,6 +161,11 @@ impl Probe { } impl Actor for Probe { async fn started(&mut self, ctx: &Context) { + // A stop gate slows stopped(); on a parent's shutdown it runs only + // for a trapping actor (v0.8, E3). + if self.stop_gate.is_some() { + ctx.trap_exit(true); + } if let Some(gate) = &self.start_gate { gate.notified().await; } @@ -192,38 +197,52 @@ impl Message for Cmd { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, cmd: Cmd, ctx: &Context) -> u64 { - let c = &self.counters; - match cmd { - Cmd::Bump => c.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1, - Cmd::Read => c.effects.load(Ordering::SeqCst) as u64, - Cmd::Watch(target) => { - ctx.monitor(&target); - c.watched.fetch_add(1, Ordering::SeqCst) as u64 - } - Cmd::Link(target) => { - ctx.link(&target); - c.linked.fetch_add(1, Ordering::SeqCst) as u64 - } - Cmd::Trap => { - ctx.trap_exit(true); - 0 - } - Cmd::Boom => panic!("cancellation fixture crash"), - Cmd::Helper(gate) => { - let task = ctx - .spawn_helper(async move { gate.notified().await }) - .unwrap(); - // Drop the ScopedTask: the helper stays owned by the actor scope. - drop(task); - 0 + async fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __reply_value: u64 = { + let c = &self.counters; + match cmd { + Cmd::Bump => c.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1, + Cmd::Read => c.effects.load(Ordering::SeqCst) as u64, + Cmd::Watch(target) => { + ctx.monitor(&target); + c.watched.fetch_add(1, Ordering::SeqCst) as u64 + } + Cmd::Link(target) => { + ctx.link(&target); + c.linked.fetch_add(1, Ordering::SeqCst) as u64 + } + Cmd::Trap => { + ctx.trap_exit(true); + 0 + } + Cmd::Boom => panic!("cancellation fixture crash"), + Cmd::Helper(gate) => { + let task = ctx + .spawn_helper(async move { gate.notified().await }) + .unwrap(); + // Drop the ScopedTask: the helper stays owned by the actor scope. + drop(task); + 0 + } } - } + }; + reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.counters.downs.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } @@ -232,8 +251,15 @@ pub trait Bumper: Send + Sync { fn bump(&self) -> Response; } impl Handler for Probe { - async fn handle(&mut self, _: bumper::Bump, _: &Context) -> u64 { - self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 + async fn handle( + &mut self, + _: bumper::Bump, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = + { self.counters.effects.fetch_add(1, Ordering::SeqCst) as u64 + 1 }; + reply.reply(__reply_value); } } @@ -432,7 +458,9 @@ async fn monitor_and_link_setup_requests_are_cancel_safe() { fx.counters.watched.load(Ordering::SeqCst) }; assert_eq!(registered, usize::from(n > 0), "{op}: N={n}"); - target.child_handle().shutdown(); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!( spawned_rt::tasks::timeout(LIMIT, target.wait_exit()) .await @@ -557,17 +585,13 @@ async fn group_ready_wait_is_cancel_safe() { // --------------------------------------------------------------------------- fn config() -> SupervisorConfig { - SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } /// SupervisorBuilder::start: a dropped startup requests cleanup of every @@ -785,19 +809,25 @@ async fn system_shutdown_is_cancel_safe() { probe.stop_gate = Some(gate.clone()); let actor = fx.spawn(probe); within!(actor.ready()).await.unwrap(); - let report = fx.system.as_ref().unwrap().shutdown(Duration::ZERO).await; + // A short grace, not zero: the trapping probe must dispatch the + // parent's exit and enter stopped() before the kill at grace + // (E3). With zero grace the kill can land first and skip + // stopped() (E1, decision 2), leaving nothing stuck to wait for. + let report = fx.system.as_ref().unwrap().shutdown(PARK).await; assert!(!report.complete); let outcome = attempt(report.wait(LIMIT), n, PARK).await; assert!( !matches!(outcome, Attempt::Done(ref r) if r.complete), "{op}" ); + // The kill at grace landed while stopped() still runs (E1). + spawned_rt::tasks::sleep(Duration::from_millis(50)).await; gate.notify_one(); assert_eq!( spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) .await .unwrap(), - ExitReason::Shutdown + ExitReason::Killed ); fx.finish(&[]).await; outcome.finished() diff --git a/concurrency/tests/cfg_gated_protocol.rs b/concurrency/tests/cfg_gated_protocol.rs index 7bf8afc50..8134f5430 100644 --- a/concurrency/tests/cfg_gated_protocol.rs +++ b/concurrency/tests/cfg_gated_protocol.rs @@ -26,8 +26,13 @@ struct MixedActor { impl Actor for MixedActor {} impl Handler for MixedActor { - async fn handle(&mut self, _msg: mixed_protocol::GetValue, _ctx: &Context) -> u64 { - self.value + async fn handle( + &mut self, + _msg: mixed_protocol::GetValue, + reply: spawned_concurrency::PendingReply<'_, u64>, + _ctx: &Context, + ) { + reply.reply(self.value); } } @@ -36,10 +41,14 @@ impl Handler for MixedActor { async fn handle( &mut self, _msg: mixed_protocol::TestOnlyIncrement, + reply: spawned_concurrency::PendingReply<'_, u64>, _ctx: &Context, - ) -> u64 { - self.value += 1; - self.value + ) { + let __reply_value: u64 = { + self.value += 1; + self.value + }; + reply.reply(__reply_value); } } diff --git a/concurrency/tests/cohort_trace.rs b/concurrency/tests/cohort_trace.rs index 99657cd8d..048324ea6 100644 --- a/concurrency/tests/cohort_trace.rs +++ b/concurrency/tests/cohort_trace.rs @@ -18,7 +18,12 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("trace cohort failure"); } } @@ -79,17 +84,17 @@ async fn trace_cohort_selection_and_temporary_retirement() { let system = ActorSystem::new().unwrap(); let group = system.async_group("cohort-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let id = tree.trace_id(); let mut children = Vec::new(); @@ -140,11 +145,11 @@ async fn trace_superseded_round_with_parked_temporary_tail() { let group = system.async_group("cohort-superseded-trace", 2).unwrap(); let entered = Arc::new(tokio::sync::Notify::new()); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30), + ) .unwrap(); let id = tree.trace_id(); let mut children = Vec::new(); @@ -213,10 +218,7 @@ async fn trace_normally_retired_transient_is_skipped_by_later_cohorts() { let system = ActorSystem::new().unwrap(); let group = system.async_group("retired-transient-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - ..SupervisorConfig::default() - }) + .supervisor(SupervisorConfig::default().strategy(strategy)) .unwrap(); let mut children = Vec::new(); for i in 0..3 { @@ -271,7 +273,12 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("trace cohort failure"); } } @@ -287,17 +294,17 @@ async fn trace_failed_replacement_starts_a_new_round() { let system = ActorSystem::new().unwrap(); let group = system.async_group("cohort-pre-ready-trace", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 30, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(30) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let id = tree.trace_id(); tree.add(ChildSpec::new("a", &group, |_| async { diff --git a/concurrency/tests/context_exit.rs b/concurrency/tests/context_exit.rs index b2da17ac0..817b883bf 100644 --- a/concurrency/tests/context_exit.rs +++ b/concurrency/tests/context_exit.rs @@ -20,9 +20,15 @@ fn native_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - fn handle(&mut self, exit: Exit, ctx: &Context) { + fn handle( + &mut self, + exit: Exit, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop_with_reason(exit.0); ctx.stop(); + reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { @@ -50,9 +56,15 @@ async fn async_explicit_exit_is_cooperative_and_first_reason_wins() { } } impl Handler for Worker { - async fn handle(&mut self, exit: Exit, ctx: &Context) { + async fn handle( + &mut self, + exit: Exit, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop_with_reason(exit.0); ctx.stop(); + reply.reply(()); } } for reason in [ExitReason::Shutdown, ExitReason::Kill] { diff --git a/concurrency/tests/coverage_trace.rs b/concurrency/tests/coverage_trace.rs index d22a32ae0..d3cdd8f3c 100644 --- a/concurrency/tests/coverage_trace.rs +++ b/concurrency/tests/coverage_trace.rs @@ -37,17 +37,15 @@ mod tree_record; const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ) } fn mode_name(mode: RestartMode) -> &'static str { @@ -94,7 +92,12 @@ impl Message for Crash { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected child failure"); } } @@ -151,39 +154,48 @@ async fn first_attempt_abandoned_when_startup_is_dropped() { write_trace("first_attempt_closed", trace_id, &modes, 5); } -/// `T|Close|..|backoff|starting|open`, then `BackoffDone/closed` and -/// `SupStartDone/notready` with a child in backoff: the first child's -/// construction fails during initial startup and, while it waits out its -/// backoff, the caller gives up on the startup (drops the start future). +/// `T|Close|..|deferred|starting|open`, then `Undefer` and `Decide/closed`: +/// a ready child's exit during the initial start is deferred (S1) and the +/// caller gives up on the startup (drops the start future) while it waits. +/// (v0.7 reached `T|Close|..|backoff|starting|open` here through a retried +/// first start, which v0.8 no longer makes.) #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn startup_cancelled_while_first_child_backs_off() { +async fn startup_cancelled_while_a_ready_exit_is_deferred() { let _scenario = FailScenario::setup(); let system = ActorSystem::new().unwrap(); - let group = system.async_group("cov-start-backoff", 1).unwrap(); + let group = system.async_group("cov-start-deferred", 2).unwrap(); let modes = [RestartMode::Permanent, RestartMode::Transient]; - let mut policy = config(5); - policy.backoff.initial = Duration::from_secs(60); - policy.backoff.maximum = Duration::from_secs(60); - let mut tree = system.supervisor(policy).unwrap(); - let flaky = tree - .add(ChildSpec::new("flaky", &group, |ctx| async move { - if ctx.generation() == 1 { - Err("first construction fails".to_string()) - } else { - Ok(Worker) - } - })) + let (_open, gate) = tokio::sync::watch::channel(false); + let mut tree = system.supervisor(config(5)).unwrap(); + let early = tree.add(worker("early", &group)).unwrap(); + let later = tree + .add( + ChildSpec::new("later", &group, move |_| { + let mut gate = gate.clone(); + async move { + let _ = gate.wait_for(|open| *open).await; + Ok::<_, String>(Worker) + } + }) + .restart(modes[1]), + ) .unwrap(); - let later = tree.add(worker("later", &group).restart(modes[1])).unwrap(); let trace_id = tree.trace_id(); let start = tokio::spawn(tree.start()); - until("backoff", || flaky.snapshot().state == ChildState::Backoff).await; + until("the later child under construction", || { + later.snapshot().state == ChildState::Starting + }) + .await; + crash(&early).await; + until("the deferred exit", || { + lifecycle_trace::recorded(trace_id, "DeferExit") + }) + .await; start.abort(); assert!(start.await.is_err()); assert!(system.shutdown(LIMIT).await.complete); - assert_eq!(flaky.snapshot().generation, 1); - assert_eq!(later.snapshot().generation, 0); - write_trace("startup_cancelled_during_backoff", trace_id, &modes, 5); + assert_eq!(early.snapshot().generation, 1); + write_trace("startup_cancelled_while_deferred", trace_id, &modes, 5); } /// Shutdown begins while a restart is under construction and its deadline @@ -242,6 +254,11 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(release) = self.release.take() { let _ = release.await; @@ -293,15 +310,21 @@ impl Actor for QuitInStarted { } } impl Handler for QuitInStarted { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected child failure"); } } -/// A ready child crashes and restarts while a later sibling is still being -/// constructed during initial startup (`T|SelfExit/Panic|..|starting|open`). +/// A ready child crashes while a later sibling is still being constructed +/// during initial startup (`T|SelfExit/Panic|..|starting|open`); its exit is +/// deferred (S1) and it restarts once the tree commits ready. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn ready_child_restarts_while_a_sibling_is_still_starting() { +async fn ready_child_exit_deferred_while_a_sibling_is_still_starting() { let _scenario = FailScenario::setup(); let system = ActorSystem::new().unwrap(); let group = system.async_group("cov-start-sibling", 2).unwrap(); @@ -321,22 +344,28 @@ async fn ready_child_restarts_while_a_sibling_is_still_starting() { .restart(modes[1]), ) .unwrap(); + let trace_id = tree.trace_id(); let start = tokio::spawn(tree.start()); until("the late child under construction", || { late.snapshot().state == ChildState::Starting }) .await; crash(&early).await; - tokio::time::timeout(LIMIT, early.wait_for_generation(1)) - .await - .unwrap() - .unwrap(); + until("the deferred exit", || { + lifecycle_trace::recorded(trace_id, "DeferExit") + }) + .await; + assert_eq!(early.snapshot().generation, 1); open.send_replace(true); let supervisor = tokio::time::timeout(LIMIT, start) .await .unwrap() .unwrap() .unwrap(); + tokio::time::timeout(LIMIT, early.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); assert_eq!(early.snapshot().generation, 2); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("restart_during_startup", supervisor.trace_id(), &modes, 5); diff --git a/concurrency/tests/cross_backend.proptest-regressions b/concurrency/tests/cross_backend.proptest-regressions new file mode 100644 index 000000000..c10357e72 --- /dev/null +++ b/concurrency/tests/cross_backend.proptest-regressions @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 7fd0a8196070dc964f5995916f8f5422ab5c0049bb709a8c539c63e90997041c # shrinks to endings = [Stop], double = [false] diff --git a/concurrency/tests/cross_backend.rs b/concurrency/tests/cross_backend.rs index c928fdb95..76608b5c4 100644 --- a/concurrency/tests/cross_backend.rs +++ b/concurrency/tests/cross_backend.rs @@ -103,6 +103,9 @@ struct State { ticks_seen: u32, pulses_wanted: u32, pulses_seen: u32, + /// Trap exits from started(): v0.8 runs stopped() on a parent's + /// (supervisor or system) shutdown only for a trapping actor (E3). + trap: bool, } impl State { fn new(index: u32, log: Log) -> Self { @@ -115,8 +118,13 @@ impl State { ticks_seen: 0, pulses_wanted: 0, pulses_seen: 0, + trap: false, } } + fn trapping(mut self) -> Self { + self.trap = true; + self + } fn echo(&mut self, v: u64) -> Reply { self.handled += 1; Reply::Value(v.wrapping_mul(1_000) + u64::from(self.index) * 10 + self.handled) @@ -144,6 +152,11 @@ mod tasks_node { pub interval: Option, } impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.state.trap { + ctx.trap_exit(true); + } + } async fn stopped(&mut self, _: &Context) { self.state.log.push(Obs::Stopped(self.state.index)); } @@ -156,77 +169,96 @@ mod tasks_node { } } impl Handler for Node { - async fn handle(&mut self, cmd: Cmd, ctx: &Context) -> Reply { - match cmd { - Cmd::Echo(v) => self.state.echo(v), - Cmd::Chain(v) => match self.peers.first() { - None => self.state.leaf(v), - Some(next) => next - .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) - .await - .unwrap_or(Reply::Failed), - }, - Cmd::FanOut(v) => { - let calls = - self.peers.iter().enumerate().map(|(i, peer)| { + async fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::PendingReply<'_, Reply>, + ctx: &Context, + ) { + let __reply_value: Reply = async { + match cmd { + Cmd::Echo(v) => self.state.echo(v), + Cmd::Chain(v) => match self.peers.first() { + None => self.state.leaf(v), + Some(next) => next + .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) + .await + .unwrap_or(Reply::Failed), + }, + Cmd::FanOut(v) => { + let calls = self.peers.iter().enumerate().map(|(i, peer)| { peer.request_with_timeout(Cmd::Echo(v + i as u64), LIMIT) }); - let mut values = Vec::new(); - for reply in futures::future::join_all(calls).await { - match reply { - Ok(Reply::Value(v)) => values.push(v), - _ => return Reply::Failed, + let mut values = Vec::new(); + for reply in futures::future::join_all(calls).await { + match reply { + Ok(Reply::Value(v)) => values.push(v), + _ => return Reply::Failed, + } } + Reply::Values(values) } - Reply::Values(values) - } - Cmd::Work { sender, seq } => { - self.state.work.push((sender, seq)); - Reply::Unit - } - Cmd::Log => Reply::Log(self.state.work.clone()), - Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), - Cmd::Link(target) => { - ctx.link(&target); - Reply::Unit - } - Cmd::Trap => { - ctx.trap_exit(true); - Reply::Unit - } - Cmd::Boom => panic!("cross-backend injected panic"), - Cmd::Stop => { - ctx.stop(); - Reply::Unit - } - Cmd::Timers { after, pulses } => { - self.state.ticks_wanted = after; - self.state.pulses_wanted = pulses; - for n in 0..after { - send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + Cmd::Work { sender, seq } => { + self.state.work.push((sender, seq)); + Reply::Unit + } + Cmd::Log => Reply::Log(self.state.work.clone()), + Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), + Cmd::Link(target) => { + ctx.link(&target); + Reply::Unit } - if pulses > 0 { - self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + Cmd::Trap => { + ctx.trap_exit(true); + Reply::Unit } - if self.state.timers_done() { + Cmd::Boom => panic!("cross-backend injected panic"), + Cmd::Stop => { ctx.stop(); + Reply::Unit + } + Cmd::Timers { after, pulses } => { + self.state.ticks_wanted = after; + self.state.pulses_wanted = pulses; + for n in 0..after { + send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + } + if pulses > 0 { + self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + } + if self.state.timers_done() { + ctx.stop(); + } + Reply::Unit } - Reply::Unit } } + .await; + reply.reply(__reply_value); } } impl Handler for Node { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); + reply.reply(()); } } impl Handler for Node { - async fn handle(&mut self, Tick(n): Tick, ctx: &Context) { + async fn handle( + &mut self, + Tick(n): Tick, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -235,25 +267,35 @@ mod tasks_node { if self.state.timers_done() { ctx.stop(); } + reply.reply(()); } } impl Handler for Node { - async fn handle(&mut self, _: Pulse, ctx: &Context) { - if self.state.pulses_seen == self.state.pulses_wanted { - return; // A tick already in flight when the interval was cancelled. - } - self.state.pulses_seen += 1; - self.state.log.push(Obs::Pulse { - node: self.state.index, - }); - if self.state.pulses_seen == self.state.pulses_wanted { - if let Some(timer) = self.interval.take() { - timer.cancellation_token.cancel(); + async fn handle( + &mut self, + _: Pulse, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __reply_value: () = async { + if self.state.pulses_seen == self.state.pulses_wanted { + return; // A tick already in flight when the interval was cancelled. + } + self.state.pulses_seen += 1; + self.state.log.push(Obs::Pulse { + node: self.state.index, + }); + if self.state.pulses_seen == self.state.pulses_wanted { + if let Some(timer) = self.interval.take() { + timer.cancellation_token.cancel(); + } + } + if self.state.timers_done() { + ctx.stop(); } } - if self.state.timers_done() { - ctx.stop(); - } + .await; + reply.reply(__reply_value); } } } @@ -269,6 +311,11 @@ mod threads_node { pub interval: Option, } impl Actor for Node { + fn started(&mut self, ctx: &Context) { + if self.state.trap { + ctx.trap_exit(true); + } + } fn stopped(&mut self, _: &Context) { self.state.log.push(Obs::Stopped(self.state.index)); } @@ -281,79 +328,103 @@ mod threads_node { } } impl Handler for Node { - fn handle(&mut self, cmd: Cmd, ctx: &Context) -> Reply { - match cmd { - Cmd::Echo(v) => self.state.echo(v), - Cmd::Chain(v) => match self.peers.first() { - None => self.state.leaf(v), - Some(next) => next - .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) - .unwrap_or(Reply::Failed), - }, - Cmd::FanOut(v) => { - // Fan out every request before collecting any reply. - let pending: Vec<_> = self - .peers - .iter() - .enumerate() - .map(|(i, peer)| peer.request_raw(Cmd::Echo(v + i as u64))) - .collect(); - let mut values = Vec::new(); - for rx in pending { - match rx.map(|rx| rx.recv_timeout(LIMIT)) { - Ok(Ok(Reply::Value(v))) => values.push(v), - _ => return Reply::Failed, + fn handle( + &mut self, + cmd: Cmd, + reply: spawned_concurrency::PendingReply<'_, Reply>, + ctx: &Context, + ) { + let __reply_value: Reply = (|| { + match cmd { + Cmd::Echo(v) => self.state.echo(v), + Cmd::Chain(v) => match self.peers.first() { + None => self.state.leaf(v), + Some(next) => next + .request_with_timeout(Cmd::Chain(chain_arg(v, self.state.index)), LIMIT) + .unwrap_or(Reply::Failed), + }, + Cmd::FanOut(v) => { + // Fan out every request before collecting any reply. + let pending: Vec<_> = self + .peers + .iter() + .enumerate() + .map(|(i, peer)| { + peer.send_request( + Cmd::Echo(v + i as u64), + spawned_concurrency::Deadline::after(LIMIT), + ) + }) + .collect(); + let mut values = Vec::new(); + for rx in pending { + match rx.map(|rx| rx.recv()) { + Ok(Ok(Reply::Value(v))) => values.push(v), + _ => return Reply::Failed, + } } + Reply::Values(values) } - Reply::Values(values) - } - Cmd::Work { sender, seq } => { - self.state.work.push((sender, seq)); - Reply::Unit - } - Cmd::Log => Reply::Log(self.state.work.clone()), - Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), - Cmd::Link(target) => { - ctx.link(&target); - Reply::Unit - } - Cmd::Trap => { - ctx.trap_exit(true); - Reply::Unit - } - Cmd::Boom => panic!("cross-backend injected panic"), - Cmd::Stop => { - ctx.stop(); - Reply::Unit - } - Cmd::Timers { after, pulses } => { - self.state.ticks_wanted = after; - self.state.pulses_wanted = pulses; - for n in 0..after { - send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + Cmd::Work { sender, seq } => { + self.state.work.push((sender, seq)); + Reply::Unit + } + Cmd::Log => Reply::Log(self.state.work.clone()), + Cmd::Watch(target) => Reply::Monitor(ctx.monitor(&target)), + Cmd::Link(target) => { + ctx.link(&target); + Reply::Unit } - if pulses > 0 { - self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + Cmd::Trap => { + ctx.trap_exit(true); + Reply::Unit } - if self.state.timers_done() { + Cmd::Boom => panic!("cross-backend injected panic"), + Cmd::Stop => { ctx.stop(); + Reply::Unit + } + Cmd::Timers { after, pulses } => { + self.state.ticks_wanted = after; + self.state.pulses_wanted = pulses; + for n in 0..after { + send_after(TICK_STEP * (n + 1), ctx.clone(), Tick(n)); + } + if pulses > 0 { + self.interval = Some(send_interval(TICK_STEP, ctx.clone(), Pulse)); + } + if self.state.timers_done() { + ctx.stop(); + } + Reply::Unit } - Reply::Unit } - } + })(); + reply.reply(__reply_value); } } impl Handler for Node { - fn handle(&mut self, down: Down, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.state.log.push(Obs::Down { observer: self.state.index, monitor: down.monitor_ref, reason: down.reason, }); + reply.reply(()); } } impl Handler for Node { - fn handle(&mut self, Tick(n): Tick, ctx: &Context) { + fn handle( + &mut self, + Tick(n): Tick, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { self.state.ticks_seen += 1; self.state.log.push(Obs::Tick { node: self.state.index, @@ -362,25 +433,34 @@ mod threads_node { if self.state.timers_done() { ctx.stop(); } + reply.reply(()); } } impl Handler for Node { - fn handle(&mut self, _: Pulse, ctx: &Context) { - if self.state.pulses_seen == self.state.pulses_wanted { - return; - } - self.state.pulses_seen += 1; - self.state.log.push(Obs::Pulse { - node: self.state.index, - }); - if self.state.pulses_seen == self.state.pulses_wanted { - if let Some(timer) = self.interval.take() { - timer.cancellation_token.cancel(); + fn handle( + &mut self, + _: Pulse, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __reply_value: () = (|| { + if self.state.pulses_seen == self.state.pulses_wanted { + return; } - } - if self.state.timers_done() { - ctx.stop(); - } + self.state.pulses_seen += 1; + self.state.log.push(Obs::Pulse { + node: self.state.index, + }); + if self.state.pulses_seen == self.state.pulses_wanted { + if let Some(timer) = self.interval.take() { + timer.cancellation_token.cancel(); + } + } + if self.state.timers_done() { + ctx.stop(); + } + })(); + reply.reply(__reply_value); } } } @@ -724,9 +804,9 @@ fn end(env: &Env, actor: &Ref, ending: Ending) -> ExitReason { Ending::Stop => assert_eq!(actor.request(Cmd::Stop).unwrap(), Reply::Unit), Ending::Panic => assert!(matches!( actor.request(Cmd::Boom), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )), - Ending::Shutdown => child.shutdown(), + Ending::Shutdown => child.stop_with_reason(spawned_concurrency::ExitReason::Shutdown), } env.wait_exit(&child) } @@ -838,7 +918,7 @@ fn monitor_workload(env: &mut Env, endings: &[Ending], double: &[bool]) -> Monit for (target, &ending) in targets.iter().zip(endings) { end(env, target, ending); } - // Watching an already-dead target delivers Down immediately. + // Watching an already-dead target delivers one Down { NoProc } at once (E4). monitors.push(vec![watch(&targets[0])]); let expected: usize = monitors.iter().map(Vec::len).sum(); let count = |events: &[Obs]| { @@ -921,22 +1001,21 @@ fn affected(strategy: SupervisorStrategy, n: usize, crashed: usize) -> std::ops: SupervisorStrategy::OneForOne => crashed..crashed + 1, SupervisorStrategy::OneForAll => 0..n, SupervisorStrategy::RestForOne => crashed..n, + other => unreachable!("strategy {other:?} not under test"), } } fn supervisor_config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 10_000, - restart_window: Duration::from_secs(60), - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(10_000) + .restart_window(Duration::from_secs(60)) + .backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } fn supervisor_workload( env: &mut Env, @@ -959,7 +1038,7 @@ fn supervisor_workload( let log = log.clone(); tree.add(ChildSpec::new(format!("c{i}"), &group, move |_| { calls[i].fetch_add(1, Ordering::SeqCst); - let state = State::new(i as u32, log.clone()); + let state = State::new(i as u32, log.clone()).trapping(); async move { Ok::<_, String>(tasks_node::Node { state, @@ -1031,8 +1110,13 @@ struct ShutdownOutcome { fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) -> ShutdownOutcome { let log = Recorder::new(); let mut ids = Vec::new(); + // Trapping, so stopped() records the system's shutdown order (E3). + let trap = |actor: Ref| { + assert_eq!(actor.request(Cmd::Trap).unwrap(), Reply::Unit); + actor.child().id() + }; for i in 0..before { - ids.push(env.start(i, vec![], &log).child().id()); + ids.push(trap(env.start(i, vec![], &log))); } let group = env.group.clone().unwrap(); let system = env.system.take().unwrap(); @@ -1042,7 +1126,7 @@ fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) for i in 0..sup_children { let log = log.clone(); tree.add(ChildSpec::new(format!("s{i}"), &group, move |_| { - let state = State::new(100 + i, log.clone()); + let state = State::new(100 + i, log.clone()).trapping(); async move { Ok::<_, String>(tasks_node::Node { state, @@ -1055,7 +1139,7 @@ fn shutdown_workload(env: &mut Env, before: u32, sup_children: u32, after: u32) } let supervisor = (sup_children > 0).then(|| env.bounded(tree.start()).unwrap().unwrap()); for i in 0..after { - ids.push(env.start(200 + i, vec![], &log).child().id()); + ids.push(trap(env.start(200 + i, vec![], &log))); } // ActorSystem::shutdown owns ordering; Env::finish must not stop actors first. env.started.clear(); @@ -1170,7 +1254,7 @@ proptest! { for (i, &ending) in endings.iter().enumerate() { prop_assert!(outcome.downs[i].iter().all(|r| *r == expected_reason(ending))); } - prop_assert_eq!(&outcome.downs[endings.len()], &vec![expected_reason(endings[0])]); + prop_assert_eq!(&outcome.downs[endings.len()], &vec![ExitReason::NoProc]); } #[test] @@ -1281,10 +1365,16 @@ mod early_stop { } } impl spawned_concurrency::tasks::Handler for Hooks { - async fn handle(&mut self, block: Block, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + block: Block, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &spawned_concurrency::tasks::Context, + ) { // Deliberately blocks the only async worker of the group. let _ = block.1.send(()); let _ = block.0.lock().unwrap().recv_timeout(LIMIT); + reply.reply(()); } } impl spawned_concurrency::threads::Actor for Hooks { diff --git a/concurrency/tests/deadline_service.rs b/concurrency/tests/deadline_service.rs new file mode 100644 index 000000000..4e31c9908 --- /dev/null +++ b/concurrency/tests/deadline_service.rs @@ -0,0 +1,135 @@ +//! Review P0 (Codex 2): user payload dropped on the process-wide deadline +//! service thread must not end the service. A job's deadline takes its +//! terminal notification while the work runs; the job's scheduled emit is +//! then discarded at its instant on the service thread, and its message's +//! destructor panics. Another actor's request deadline must still fire. +//! +//! The service is process-wide, so the scenario runs in a child process. +#![cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +use spawned_concurrency::{ + message::Message, + tasks::{Actor, ActorStart, Context, Handler, Instant, JobOptions, JobOutcome}, + ActorError, Deadline, PendingReply, ReplyTo, +}; +use std::time::Duration; +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +/// A message whose destructor panics while armed. +struct Bomb(bool); +impl Message for Bomb { + type Result = (); +} +impl Drop for Bomb { + fn drop(&mut self) { + if self.0 { + panic!("scheduled emit destructor"); + } + } +} + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Starts the job; parks requests (deferred, never answered). +#[derive(Default)] +struct Owner(Vec>); +impl Actor for Owner {} +impl Handler for Owner { + async fn handle(&mut self, mut msg: Bomb, _: PendingReply<'_, ()>, _: &Context) { + msg.0 = false; // delivered: not the scenario under test + } +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} +struct Park; +impl Message for Park { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.0.push(reply.defer().unwrap()); + } +} + +#[test] +fn deadline_service_survives_a_panicking_payload_destructor() { + subprocess::run("deadline_service_child", "deadline service survived"); +} + +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn deadline_service_child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let owner = Owner::default().start(); + spawned_concurrency::__private::in_turn(&owner, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(40); + job.emit_at_blocking(at, Bomb(true)).unwrap(); + std::thread::sleep(Duration::from_millis(150)); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + }) + .unwrap() + .await + .unwrap(); + // The deadline (20 ms) took the notification; the emit is discarded + // at 40 ms on the service thread and its destructor panics. + tokio::time::sleep(Duration::from_millis(100)).await; + + // Another actor's deadlines, serviced by the same thread. + let other = Owner::default().start(); + let parked = other + .send_request(Park, Deadline::after(Duration::from_millis(50))) + .unwrap(); + match tokio::time::timeout(Duration::from_secs(5), parked).await { + Ok(Err(ActorError::RequestTimeout)) => {} + Ok(other) => panic!("unexpected outcome {other:?}"), + Err(_) => panic!("the deadline service stopped: a parked request never expired"), + } + // A job deadline still fires too. + let fired = spawned_concurrency::__private::in_turn(&other, |_, ctx| { + let (tx, rx) = std::sync::mpsc::channel(); + let opts = JobOptions::new().deadline(Duration::from_millis(20)); + ctx.pipe_blocking( + opts, + |_| std::thread::sleep(Duration::from_millis(300)), + move |outcome: JobOutcome<()>| { + let _ = tx.send(outcome.is_err()); + Done + }, + ) + .unwrap(); + rx + }) + .unwrap() + .await + .unwrap(); + let deadline = + tokio::task::spawn_blocking(move || fired.recv_timeout(Duration::from_secs(5))) + .await + .unwrap(); + assert_eq!(deadline, Ok(true), "a job deadline did not fire"); + other.context().stop(); + other.join().await; + owner.context().stop(); + owner.join().await; + }); + println!("deadline service survived"); +} diff --git a/concurrency/tests/death_boundary.rs b/concurrency/tests/death_boundary.rs new file mode 100644 index 000000000..7d10bf46a --- /dev/null +++ b/concurrency/tests/death_boundary.rs @@ -0,0 +1,868 @@ +//! The death point D (docs/design/semantics.md, `Lifecycle!NoOrphanRequest`): +//! every queued request and every deferred reply is answered with the final +//! reason at D, before any payload or state destructor runs and whatever +//! drives the death (the run loop, or the runtime dropping the actor). Each +//! test blocks a destructor and checks that the callers are answered while +//! it is still blocked. +#![cfg(not(spawned_sim))] // blocking destructors and OS threads +use spawned_concurrency::{message::Message, ActorError, ExitReason}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +const LIMIT: Duration = Duration::from_secs(5); + +/// A payload whose destructor blocks until released, and reports that it +/// started. It never releases itself within an assertion window (`LIMIT`): +/// a test that answers only after destruction fails, never passes late. +struct Gated { + started: Option>, + release: Option>, +} +impl Drop for Gated { + fn drop(&mut self) { + if let (Some(started), Some(release)) = (self.started.take(), self.release.take()) { + let _ = started.send(()); + let _ = release.recv_timeout(LIMIT * 6); + } + } +} +/// A gated payload, its "destructor started" signal and its release. +fn gated() -> (Gated, mpsc::Receiver<()>, mpsc::Sender<()>) { + let (started_tx, started) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + ( + Gated { + started: Some(started_tx), + release: Some(release_rx), + }, + started, + release, + ) +} +fn inert() -> Gated { + Gated { + started: None, + release: None, + } +} + +/// A request carrying a payload. +struct Queued(#[allow(dead_code)] Gated); +impl Message for Queued { + type Result = (); +} +/// Blocks the handler until released. +struct Block(mpsc::Receiver<()>, mpsc::Sender<()>); +impl Message for Block { + type Result = (); +} + +/// A payload whose destructor waits (at most 3 s) for a queued caller's +/// answer, and records whether it arrived: destroyed before the death point +/// answers that caller, it waits in vain. +struct WaitsForAnswer { + caller: std::sync::Arc>>>, + answered: std::sync::Arc, +} +impl Drop for WaitsForAnswer { + fn drop(&mut self) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + let mut caller = self.caller.lock().unwrap(); + if let Some(receiver) = caller.as_mut() { + if receiver.try_recv().is_some() { + self.answered + .store(true, std::sync::atomic::Ordering::SeqCst); + return; + } + } + drop(caller); + std::thread::sleep(Duration::from_millis(1)); + } + } +} +/// A job's message carrying such a payload. +struct Waiting(#[allow(dead_code)] WaitsForAnswer); +impl Message for Waiting { + type Result = (); +} + +/// Wait until `poll` yields, at most `LIMIT`. +fn eventually(what: &str, mut poll: impl FnMut() -> Option) -> T { + let began = Instant::now(); + loop { + if let Some(value) = poll() { + return value; + } + assert!(began.elapsed() < LIMIT, "{what}"); + std::thread::sleep(Duration::from_millis(2)); + } +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::{PendingReply, ReplyTo}; + use std::future::Future; + use std::sync::Arc; + + /// Defers and keeps the token in the actor's state. + struct Park; + impl Message for Park { + type Result = (); + } + + struct Owner { + _state: Gated, + parked: Vec>, + } + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, _: Park, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.push(reply.defer().unwrap()); + } + } + + /// Defers and hands the token out. + struct HandOut(mpsc::Sender>); + impl Message for HandOut { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: HandOut, reply: PendingReply<'_, ()>, _: &Context) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } + } + + impl Handler for Owner { + async fn handle(&mut self, _: Waiting, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review round 2 (Codex 1): the owner's exit cancels its jobs without + /// destroying their payloads; a scheduled emit's message and a held + /// result are destroyed after the death point answered every caller. + /// Cancellation used to destroy them first: a destructor waiting for a + /// queued caller's answer waited for an answer that could not come. + #[test] + fn job_payloads_are_destroyed_after_every_caller_is_answered() { + use spawned_concurrency::tasks::JobOptions; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + for held in [false, true] { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let caller = Arc::new(Mutex::new(None)); + let answered = Arc::new(AtomicBool::new(false)); + let payload = WaitsForAnswer { + caller: caller.clone(), + answered: answered.clone(), + }; + let (scheduled_tx, scheduled) = mpsc::channel(); + let (handle_tx, handle) = mpsc::channel(); + runtime.block_on(async { + spawned_concurrency::__private::in_turn(&actor, move |_, ctx| { + let job = ctx + .pipe( + JobOptions::new(), + move |job| async move { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + if held { + // The result waits for this emit: held. + job.emit_at(far, Queued(inert())).await.unwrap(); + scheduled_tx.send(()).unwrap(); + Some(payload) + } else { + job.emit_at(far, Waiting(payload)).await.unwrap(); + scheduled_tx.send(()).unwrap(); + job.cancelled().await; + None + } + }, + |_| Queued(inert()), + ) + .unwrap(); + handle_tx.send(job).unwrap(); + }) + .unwrap() + .await + .unwrap(); + }); + scheduled.recv_timeout(LIMIT).unwrap(); + if held { + // Handshake: the work returned with its emit still + // scheduled, so its result is held (`JobCore::finish` stores + // it before execution is marked over). + let job = handle.recv_timeout(LIMIT).unwrap(); + eventually("the held job finished executing", || { + (!job.is_running()).then_some(()) + }); + } + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let queued = actor + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *caller.lock().unwrap() = Some(queued); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + answered.load(Ordering::SeqCst), + "a job payload (held: {held}) was destroyed before the queued caller was answered" + ); + } + } + + /// Holds its message (and its destructor) across a pending await. + struct Hang(#[allow(dead_code)] Gated, mpsc::Sender<()>); + impl Message for Hang { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: Hang, _: PendingReply<'_, ()>, _: &Context) { + let msg = msg; + let _ = msg.1.send(()); + std::future::pending::<()>().await; + drop(msg); + } + } + + /// Review round 2 (Codex 2, agy 1): a runtime that drops an actor while + /// a turn is pending destroys the turn (its message, the handler's + /// locals) only after the death point answered the turn's own caller and + /// every queued one. The pending lifecycle future used to be destroyed + /// first, so a blocking destructor in it left every caller waiting. + #[test] + fn a_dropped_pending_turn_is_destroyed_after_every_caller_is_answered() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (payload, destroying, release) = gated(); + let (entered_tx, entered) = mpsc::channel(); + let never = spawned_concurrency::Deadline::Never; + let (actor, mut turn_caller, mut queued) = runtime.block_on(async { + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let turn_caller = actor + .send_request(Hang(payload, entered_tx), never) + .unwrap(); + let queued = actor.send_request(Queued(inert()), never).unwrap(); + while entered.try_recv().is_err() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + (actor, turn_caller, queued) + }); + let dropper = std::thread::spawn(move || drop(runtime)); + // The pending turn's message is being destroyed (and blocks). + destroying.recv_timeout(LIMIT).unwrap(); + eventually( + "the dropped turn's caller waited for its destruction", + || stopped(turn_caller.try_recv(), &ExitReason::Killed), + ); + eventually("a queued caller waited for the turn's destruction", || { + stopped(queued.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } + + /// Review round 2 (Codex 5): the death point answers every caller before + /// any wake-up runs. The first queued caller's waker waits for the second + /// caller's answer; closing the first slot used to run that waker before + /// the second slot was closed, so the second answer had to wait. + #[test] + fn every_queued_caller_is_answered_before_any_wake_runs() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + use std::task::{Wake, Waker}; + type Receiver = spawned_concurrency::ReplyReceiver<()>; + struct WaitForSecond { + second: Mutex, + answered: AtomicBool, + } + impl Wake for WaitForSecond { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.second.lock().unwrap().try_recv().is_some() { + self.answered.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let first = actor.send_request(Queued(inert()), never).unwrap(); + let second = actor.send_request(Queued(inert()), never).unwrap(); + let waiter = Arc::new(WaitForSecond { + second: Mutex::new(second), + answered: AtomicBool::new(false), + }); + let waker = Waker::from(waiter.clone()); + let mut awaiting = Box::pin(std::future::IntoFuture::into_future(first)); + assert!(awaiting + .as_mut() + .poll(&mut std::task::Context::from_waker(&waker)) + .is_pending()); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + waiter.answered.load(Ordering::SeqCst), + "a wake-up ran before every queued caller was answered" + ); + } + + /// Review round 2 (Codex 3): `ReplyTo::overloaded()` holds its + /// death-fence permit only across the transition. Its caller's waker + /// runs after: here it stops the owner and waits for the published + /// exit. The permit used to cover the wake, so the owner's death waited + /// for the waker and the waker for the death. + #[test] + fn overloaded_wakes_the_caller_after_releasing_the_death_fence() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::{Wake, Waker}; + struct StopAndWait { + actor: spawned_concurrency::tasks::ActorRef, + exited: Arc, + } + impl Wake for StopAndWait { + fn wake(self: Arc) { + self.actor.context().stop(); + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.actor.exit_reason().is_some() { + self.exited.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (tokens, token) = mpsc::channel(); + let receiver = actor + .send_request(HandOut(tokens), spawned_concurrency::Deadline::Never) + .unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + let exited = Arc::new(AtomicBool::new(false)); + let waker = Waker::from(Arc::new(StopAndWait { + actor: actor.clone(), + exited: exited.clone(), + })); + let mut awaiting = Box::pin(std::future::IntoFuture::into_future(receiver)); + let mut cx = std::task::Context::from_waker(&waker); + assert!(awaiting.as_mut().poll(&mut cx).is_pending()); + // The wake runs on this helper, inside `overloaded()`. + std::thread::spawn(move || token.overloaded()) + .join() + .unwrap(); + assert!( + exited.load(Ordering::SeqCst), + "the owner could not die while overloaded() held its fence permit" + ); + } + + /// Pipes a job that hands its `JobCtx` to a helper and never ends. + struct ExportJob(mpsc::Sender>); + impl Message for ExportJob { + type Result = (); + } + struct JobDone; + impl Message for JobDone { + type Result = (); + } + impl Handler for Owner { + async fn handle( + &mut self, + msg: ExportJob, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { + let export = msg.0; + ctx.pipe( + spawned_concurrency::tasks::JobOptions::new(), + move |job| { + export.send(job).unwrap(); + std::future::pending::<()>() + }, + |_| JobDone, + ) + .unwrap(); + reply.reply(()); + } + } + impl Handler for Owner { + async fn handle(&mut self, _: JobDone, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Final review (Codex P1 2): the owner's exit cancels its jobs, but the + /// cancellation's callbacks (a `cancelled()` waiter's waker, arbitrary + /// code) run only after every caller is answered. Here a helper polls + /// `job.cancelled()` with a waker that waits for a queued caller's + /// answer; the exit used to run it inside cancellation, before that + /// caller could be answered. + #[test] + fn job_cancellation_wakes_after_every_caller_is_answered() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + use std::task::{Wake, Waker}; + struct WaitForCaller { + caller: Mutex>, + answered: AtomicBool, + } + impl Wake for WaitForCaller { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.caller.lock().unwrap().try_recv().is_some() { + self.answered.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (export, exported) = mpsc::channel(); + runtime.block_on(actor.request(ExportJob(export))).unwrap(); + let job = exported.recv_timeout(LIMIT).unwrap(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let caller = actor + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + let waiter = Arc::new(WaitForCaller { + caller: Mutex::new(caller), + answered: AtomicBool::new(false), + }); + // Handshake: the helper's cancelled() is registered with the waker. + let (polled_tx, polled) = mpsc::channel(); + let (done_tx, done) = mpsc::channel::<()>(); + let helper = { + let waiter = waiter.clone(); + std::thread::spawn(move || { + let waker = Waker::from(waiter); + let mut cancelled = Box::pin(job.cancelled()); + assert!(cancelled + .as_mut() + .poll(&mut std::task::Context::from_waker(&waker)) + .is_pending()); + polled_tx.send(()).unwrap(); + let _ = done.recv_timeout(LIMIT * 2); + }) + }; + polled.recv_timeout(LIMIT).unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + runtime.block_on(actor.join()); + assert!( + waiter.answered.load(Ordering::SeqCst), + "a job cancellation waker ran before every caller was answered" + ); + done_tx.send(()).unwrap(); + helper.join().unwrap(); + } + + fn stopped(result: Option>, reason: &ExitReason) -> Option<()> { + match result? { + Err(ActorError::ActorStopped(r)) if r == *reason => Some(()), + Err(other) => panic!("unexpected error {other}"), + Ok(_) => panic!("a request that was never handled was answered Ok"), + } + } + + /// Review P0 (Codex 1): requests queued behind the exit are answered + /// at D, before the first queued payload's destructor runs. The mailbox + /// used to be dropped envelope by envelope, each payload before its + /// request: a blocking destructor left its own caller and every later + /// one unanswered while it ran. + #[test] + fn queued_requests_are_answered_before_payload_destructors() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let _guard = runtime.enter(); + let actor = Owner { + _state: inert(), + parked: Vec::new(), + } + .start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let (payload, destroying, release) = gated(); + let mut first = actor + .send_request( + Queued(payload), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let mut second = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + destroying.recv_timeout(LIMIT).unwrap(); + // The first queued payload's destructor is running (blocked). + eventually("the queued callers were not answered at death", || { + stopped(first.try_recv(), &ExitReason::Normal) + }); + eventually( + "a later queued caller waited for an earlier destructor", + || stopped(second.try_recv(), &ExitReason::Normal), + ); + release.send(()).unwrap(); + runtime.block_on(actor.join()); + } + + /// Review P1 (Codex 5): a never-polled actor dropped with its runtime + /// reaches D (queued callers answered `Killed`) before its state and + /// mailbox are destroyed. Its startup payload was destroyed first. + #[test] + fn a_never_polled_actor_answers_before_its_state_is_destroyed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (state, destroying, release) = gated(); + // Spawned but never polled: the runtime is never driven. + let actor = { + let _guard = runtime.enter(); + Owner { + _state: state, + parked: Vec::new(), + } + .start() + }; + let mut queued = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let dropper = std::thread::spawn(move || drop(runtime)); + destroying.recv_timeout(LIMIT).unwrap(); + eventually("a queued caller waited for the state destructor", || { + stopped(queued.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } + + /// Review P1 (Codex 5): a running actor dropped with its runtime reaches + /// D (deferred and queued callers answered `Killed`) before its state is + /// destroyed. The state was destroyed first, and the registry closed + /// after publication. + #[test] + fn a_dropped_running_actor_answers_before_its_state_is_destroyed() { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (state, destroying, release) = gated(); + let (actor, mut parked) = runtime.block_on(async { + let actor = Owner { + _state: state, + parked: Vec::new(), + } + .start(); + let parked = actor + .send_request(Park, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + // Let the actor defer it. + actor.request(Queued(inert())).await.unwrap(); + (actor, parked) + }); + let dropper = std::thread::spawn(move || drop(runtime)); + destroying.recv_timeout(LIMIT).unwrap(); + eventually("a deferred caller waited for the state destructor", || { + stopped(parked.try_recv(), &ExitReason::Killed) + }); + release.send(()).unwrap(); + dropper.join().unwrap(); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Owner { + fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + + impl Handler for Owner { + fn handle(&mut self, _: Waiting, _: PendingReply<'_, ()>, _: &Context) {} + } + + /// Review round 2 (Codex 1), threads mode: a scheduled emit's message + /// is destroyed after the death point answered every caller. + #[test] + fn job_payloads_are_destroyed_after_every_caller_is_answered() { + use spawned_concurrency::threads::JobOptions; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + let actor = Owner.start(); + let caller = Arc::new(Mutex::new(None)); + let answered = Arc::new(AtomicBool::new(false)); + let payload = WaitsForAnswer { + caller: caller.clone(), + answered: answered.clone(), + }; + let (scheduled_tx, scheduled) = mpsc::channel(); + spawned_test::threads::in_actor(&actor, move |_, ctx| { + ctx.pipe_blocking( + JobOptions::new(), + move |job| { + let far = spawned_concurrency::Instant::now() + LIMIT * 10; + job.emit_at_blocking(far, Waiting(payload)).unwrap(); + scheduled_tx.send(()).unwrap(); + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(1)); + } + }, + |_| Queued(inert()), + ) + .unwrap(); + }); + scheduled.recv_timeout(LIMIT).unwrap(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let queued = actor + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *caller.lock().unwrap() = Some(queued); + actor.context().stop(); + unblock.send(()).unwrap(); + actor.join(); + assert!( + answered.load(Ordering::SeqCst), + "a job payload was destroyed before the queued caller was answered" + ); + } + + /// Review P0 (Codex 1), threads mode. + #[test] + fn queued_requests_are_answered_before_payload_destructors() { + let actor = Owner.start(); + let (unblock, blocked) = mpsc::channel(); + let (entered_tx, entered) = mpsc::channel(); + actor.send(Block(blocked, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let (payload, destroying, release) = gated(); + let mut first = actor + .send_request( + Queued(payload), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let mut second = actor + .send_request( + Queued(inert()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + actor.context().stop(); + unblock.send(()).unwrap(); + destroying.recv_timeout(LIMIT).unwrap(); + let answered = |rx: &mut spawned_concurrency::ReplyReceiver<()>| match rx.try_recv()? { + Err(ActorError::ActorStopped(ExitReason::Normal)) => Some(()), + other => panic!("unexpected {other:?}"), + }; + eventually("the queued callers were not answered at death", || { + answered(&mut first) + }); + eventually( + "a later queued caller waited for an earlier destructor", + || answered(&mut second), + ); + release.send(()).unwrap(); + actor.join(); + } +} + +#[cfg(feature = "async")] +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +/// Review round 2 (Codex 5): a runtime drop's "stopped() skipped" log (a +/// tracing subscriber is user code) runs after every caller is answered. It +/// used to run between the seal and the answers, so a subscriber waiting for +/// a queued caller blocked the death point. Global subscriber: own process. +#[cfg(feature = "async")] +mod subscriber_after_discharge { + use super::subprocess; + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::{PendingReply, ReplyReceiver}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Mutex; + + static QUEUED: Mutex>> = Mutex::new(None); + static ANSWERED_WHEN_LOGGED: AtomicBool = AtomicBool::new(false); + static LOGGED: AtomicBool = AtomicBool::new(false); + + /// Waits (bounded) in every event for the queued caller's answer. + struct WaitingSubscriber; + impl tracing::Subscriber for WaitingSubscriber { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn event(&self, _: &tracing::Event<'_>) { + let Some(mut queued) = QUEUED.lock().unwrap().take() else { + return; + }; + LOGGED.store(true, Ordering::SeqCst); + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if queued.try_recv().is_some() { + ANSWERED_WHEN_LOGGED.store(true, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} + } + + struct Idle; + impl Actor for Idle {} + impl Handler for Idle { + async fn handle(&mut self, _: Queued, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + + #[test] + fn the_skipped_stopped_log_runs_after_every_caller_is_answered() { + subprocess::run( + "subscriber_after_discharge::child", + "skipped-stopped log ran after the answers", + ); + } + + #[test] + #[ignore = "subprocess helper exercised by the enabled parent"] + fn child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + tracing::subscriber::set_global_default(WaitingSubscriber).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + // Never polled: dropping the runtime drops it before `stopped()`. + let actor = { + let _guard = runtime.enter(); + Idle.start() + }; + let queued = actor + .send_request(Queued(inert()), spawned_concurrency::Deadline::Never) + .unwrap(); + *QUEUED.lock().unwrap() = Some(queued); + drop(runtime); + assert!(LOGGED.load(Ordering::SeqCst), "nothing was logged"); + assert!( + ANSWERED_WHEN_LOGGED.load(Ordering::SeqCst), + "the log ran before the queued caller was answered" + ); + println!("skipped-stopped log ran after the answers"); + } +} diff --git a/concurrency/tests/dedicated_runtime.rs b/concurrency/tests/dedicated_runtime.rs index e91d20f96..b8a52873f 100644 --- a/concurrency/tests/dedicated_runtime.rs +++ b/concurrency/tests/dedicated_runtime.rs @@ -92,51 +92,59 @@ impl Message for Inspect { type Result = Observation; } impl Handler for Probe { - async fn handle(&mut self, operation: Inspect, _: &Context) -> Observation { - let root = self.assert_root(); - let handle = tokio::runtime::Handle::current(); - let workers = handle.metrics().num_workers(); - let flavor = handle.runtime_flavor(); - let worker = match operation { - Inspect::SpawnBlocking => tokio::time::timeout( - LIMIT, - tokio::task::spawn_blocking(|| thread::current().id()), - ) - .await - .unwrap() - .unwrap(), - Inspect::Yield => { - tokio::task::yield_now().await; - tokio::time::timeout(LIMIT, tokio::spawn(timed_helper())) - .await - .unwrap() - .unwrap() - } - Inspect::BlockingReceive | Inspect::BlockInPlace => { - let (tx, rx) = mpsc::channel(); - let helper = tokio::spawn(async move { - let worker = timed_helper().await; - tx.send(worker).unwrap(); - }); - // The dedicated root synchronously waits for an async helper. - // A current-thread runtime cannot make progress here, but the - // bounded receive still lets that regression terminate. - let worker = if matches!(operation, Inspect::BlockInPlace) { - tokio::task::block_in_place(|| rx.recv_timeout(LIMIT).unwrap()) - } else { - rx.recv_timeout(LIMIT).unwrap() - }; - tokio::time::timeout(LIMIT, helper).await.unwrap().unwrap(); - worker + async fn handle( + &mut self, + operation: Inspect, + reply: spawned_concurrency::PendingReply<'_, Observation>, + _: &Context, + ) { + let __reply_value: Observation = { + let root = self.assert_root(); + let handle = tokio::runtime::Handle::current(); + let workers = handle.metrics().num_workers(); + let flavor = handle.runtime_flavor(); + let worker = match operation { + Inspect::SpawnBlocking => tokio::time::timeout( + LIMIT, + tokio::task::spawn_blocking(|| thread::current().id()), + ) + .await + .unwrap() + .unwrap(), + Inspect::Yield => { + tokio::task::yield_now().await; + tokio::time::timeout(LIMIT, tokio::spawn(timed_helper())) + .await + .unwrap() + .unwrap() + } + Inspect::BlockingReceive | Inspect::BlockInPlace => { + let (tx, rx) = mpsc::channel(); + let helper = tokio::spawn(async move { + let worker = timed_helper().await; + tx.send(worker).unwrap(); + }); + // The dedicated root synchronously waits for an async helper. + // A current-thread runtime cannot make progress here, but the + // bounded receive still lets that regression terminate. + let worker = if matches!(operation, Inspect::BlockInPlace) { + tokio::task::block_in_place(|| rx.recv_timeout(LIMIT).unwrap()) + } else { + rx.recv_timeout(LIMIT).unwrap() + }; + tokio::time::timeout(LIMIT, helper).await.unwrap().unwrap(); + worker + } + }; + self.assert_root(); + Observation { + root, + worker, + workers, + flavor, } }; - self.assert_root(); - Observation { - root, - worker, - workers, - flavor, - } + reply.reply(__reply_value); } } @@ -147,9 +155,17 @@ impl Message for Ping { struct Child; impl Actor for Child {} impl Handler for Child { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }; + reply.reply(__reply_value); } } struct Nested; @@ -157,16 +173,24 @@ impl Message for Nested { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, _: Nested, _: &Context) -> u64 { - self.assert_root(); - // Default Async startup uses this actor's isolated runtime. - let child = Child.start(); - let _cleanup = StopOnDrop(child.child_handle()); - let answer = child.request_with_timeout(Ping, LIMIT).await.unwrap(); - child.child_handle().stop(); - tokio::time::timeout(LIMIT, child.join()).await.unwrap(); - self.assert_root(); - answer + async fn handle( + &mut self, + _: Nested, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.assert_root(); + // Default Async startup uses this actor's isolated runtime. + let child = Child.start(); + let _cleanup = StopOnDrop(child.child_handle()); + let answer = child.request_with_timeout(Ping, LIMIT).await.unwrap(); + child.child_handle().stop(); + tokio::time::timeout(LIMIT, child.join()).await.unwrap(); + self.assert_root(); + answer + }; + reply.reply(__reply_value); } } @@ -175,20 +199,28 @@ impl Message for Exchange { type Result = u64; } impl Handler for Probe { - async fn handle(&mut self, message: Exchange, _: &Context) -> u64 { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - self.assert_root(); - let answer = tokio::time::timeout(LIMIT, async { - let mut stream = tokio::net::TcpStream::connect(message.0).await.unwrap(); - stream.write_all(&42u64.to_le_bytes()).await.unwrap(); - let mut bytes = [0; 8]; - stream.read_exact(&mut bytes).await.unwrap(); - u64::from_le_bytes(bytes) - }) - .await - .expect("dedicated runtime I/O stalled"); - self.assert_root(); - answer + async fn handle( + &mut self, + message: Exchange, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + self.assert_root(); + let answer = tokio::time::timeout(LIMIT, async { + let mut stream = tokio::net::TcpStream::connect(message.0).await.unwrap(); + stream.write_all(&42u64.to_le_bytes()).await.unwrap(); + let mut bytes = [0; 8]; + stream.read_exact(&mut bytes).await.unwrap(); + u64::from_le_bytes(bytes) + }) + .await + .expect("dedicated runtime I/O stalled"); + self.assert_root(); + answer + }; + reply.reply(__reply_value); } } diff --git a/concurrency/tests/deferred_api.rs b/concurrency/tests/deferred_api.rs new file mode 100644 index 000000000..a09fe8344 --- /dev/null +++ b/concurrency/tests/deferred_api.rs @@ -0,0 +1,872 @@ +//! Deferred-reply API refinements from porting ethlambda and ethrex to 0.8: +//! bounds set in `#[actor]` and per `ChildSpec`, blocking jobs on Tokio's +//! blocking pool, reserve-then-spawn permits, fenced delayed emits +//! (`emit_at`), hard job deadlines, send-only manual handlers +//! (`#[send_handler]` on an impl) and `spawned_test::in_actor`. +use spawned_concurrency::message::Message; +use std::sync::{Arc, Mutex}; + +/// What the owner's handlers observed, in dispatch order. +#[derive(Default)] +pub struct Log(pub Mutex>); +impl Log { + pub fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + pub fn snapshot(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +pub struct Note(pub String); +impl Message for Note { + type Result = (); +} + +/// A send-only message whose handler returns early on one path. +pub struct Poke(pub bool); +impl Message for Poke { + type Result = (); +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Handler, Instant, JobError, JobGone, JobOptions, + JobOutcome, PipeError, + }; + use spawned_concurrency::{actor, send_handler}; + use std::time::Duration; + + /// `spawned_test::tasks::in_actor` (the dev-dependency is built without + /// its Tokio half; `testing/tests/tasks_kit.rs` covers the helper). + async fn in_actor(actor: &ActorRef, f: F) -> R + where + R: Send + 'static, + F: FnOnce(&mut Owner, &Context) -> R + Send + 'static, + { + spawned_concurrency::__private::in_turn(actor, f) + .unwrap() + .await + .unwrap() + } + + pub struct Owner { + log: Arc, + } + + #[actor(max_outstanding = 3, max_native_jobs = 2)] + impl Owner { + #[send_handler] + async fn note(&mut self, msg: Note, _: &Context) { + self.log.push(msg.0); + } + } + + #[send_handler] + impl Handler for Owner { + async fn handle(&mut self, msg: Poke, _ctx: &Context) { + if msg.0 { + return; + } + self.log.push("poked"); + } + } + + fn describe(name: &str, outcome: JobOutcome) -> Note { + Note(match outcome { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), + Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + Err(other) => format!("{name}:{other}"), + }) + } + + fn start() -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + (Owner { log: log.clone() }.start(), log) + } + + async fn wait_for(log: &Log, n: usize) -> Vec { + for _ in 0..1000 { + let entries = log.snapshot(); + if entries.len() >= n { + return entries; + } + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; + } + panic!("log never reached {n} entries: {:?}", log.snapshot()); + } + + #[test] + fn actor_macro_sets_the_bounds() { + assert_eq!(::MAX_OUTSTANDING, 3); + assert_eq!(::MAX_NATIVE_JOBS, Some(2)); + } + + #[spawned_macros::sim_test] + async fn macro_bounds_apply_and_permits_keep_the_parts() { + let (actor, _log) = start(); + let kinds = in_actor(&actor, |_, ctx| { + let a = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("first native slot"); + let b = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("second native slot"); + // max_native_jobs = 2: the third is refused, and the caller keeps + // whatever it meant to move into the job. + let third = ctx.reserve_blocking_job(JobOptions::new()).err(); + drop((a, b)); + let c = ctx.reserve_job(JobOptions::new()).expect("credit"); + let d = ctx.reserve_job(JobOptions::new()).expect("credit"); + let e = ctx.reserve_job(JobOptions::new()).expect("credit"); + // max_outstanding = 3. + let fourth = ctx.reserve_job(JobOptions::new()).err(); + drop((c, d, e)); + let again = ctx.reserve_job(JobOptions::new()).is_ok(); + (third, fourth, again, ctx.deferred_outstanding()) + }) + .await; + assert!(matches!(kinds.0, Some(PipeError::NativeBudget))); + assert!(matches!(kinds.1, Some(PipeError::Overloaded))); + assert!(kinds.2, "dropped permits return their credits"); + assert_eq!(kinds.3, 0); + actor.context().stop(); + actor.join().await; + } + + #[cfg(not(spawned_sim))] // native jobs are not simulated + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_jobs_run_on_the_tokio_blocking_pool() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let permit = ctx.reserve_blocking_job(JobOptions::new()).unwrap(); + permit.pipe_blocking( + |_| std::thread::current().name().map(str::to_owned), + |o| describe("thread", o), + ); + ctx.pipe_blocking( + JobOptions::new(), + |_| -> u32 { panic!("check boom") }, + |o| describe("panics", o), + ) + .unwrap(); + }) + .await; + let mut entries = wait_for(&log, 2).await; + entries.sort(); + assert_eq!(entries[0], "panics:panicked:check boom"); + assert!( + entries[1].starts_with("thread:ok:Some(\"tokio"), + "not on the blocking pool: {}", + entries[1] + ); + actor.context().stop(); + actor.join().await; + } + + #[cfg(not(spawned_sim))] // native jobs are not simulated + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn emit_at_arrives_late_but_before_the_terminal_message() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_blocking( + JobOptions::new(), + |job| { + let at = Instant::now() + Duration::from_millis(80); + job.emit_at_blocking(at, Note("late".into()))?; + job.emit_blocking(Note("now".into()))?; + Ok::<_, JobGone>(()) + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + let entries = wait_for(&log, 3).await; + assert_eq!(entries, ["now", "late", "job:ok:Ok(())"]); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn async_emit_at_holds_the_terminal_message() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe( + JobOptions::new(), + |job| async move { + let at = Instant::now() + Duration::from_millis(50); + job.emit_at(at, Note("late".into())).await?; + job.emit(Note("now".into())).await + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + let entries = wait_for(&log, 3).await; + assert_eq!(entries, ["now", "late", "job:ok:Ok(())"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn a_cancelled_job_fences_its_scheduled_emits() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().key("k"); + ctx.pipe( + opts.clone(), + |job| async move { + let at = Instant::now() + Duration::from_millis(60); + job.emit_at(at, Note("stale".into())).await + }, + |o| describe("old", o), + ) + .unwrap(); + }) + .await; + // The replacement cancels the previous job: its scheduled emit and + // its held terminal message are dropped. + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().key("k"); + ctx.pipe(opts, |_| async { 1 }, |o| describe("new", o)) + .unwrap(); + }) + .await; + spawned_rt::tasks::sleep(Duration::from_millis(150)).await; + assert_eq!(log.snapshot(), ["new:ok:1"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + #[cfg(not(spawned_sim))] // native jobs are not simulated + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn job_deadlines_are_hard() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe_blocking( + opts, + |job| { + std::thread::sleep(Duration::from_millis(120)); + // Past the deadline: the job is over for the owner. + job.emit_blocking(Note("too late".into())).is_err() + }, + |o| describe("slow", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 1).await, ["slow:deadline"]); + spawned_rt::tasks::sleep(Duration::from_millis(200)).await; + assert_eq!(log.snapshot(), ["slow:deadline"]); + actor.context().stop(); + actor.join().await; + } + + /// Review P1 (Codex 8, agy 1): a hard deadline overtakes a result held + /// for a scheduled emit. The run used to drop its only strong reference + /// to the deadline when the work returned, so the deadline never fired: + /// the emit and `Ok` arrived at the emit's instant instead. + #[spawned_macros::sim_test] + async fn a_deadline_overtakes_a_result_held_for_scheduled_emits() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe( + opts, + |job| async move { + let at = Instant::now() + Duration::from_millis(300); + job.emit_at(at, Note("late".into())).await.unwrap(); + 7 + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + spawned_rt::tasks::sleep(Duration::from_millis(150)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + // The scheduled emit is dropped at its instant (the job is over). + spawned_rt::tasks::sleep(Duration::from_millis(300)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + /// Review P1 (Codex 8), tasks-mode blocking job: the deadline overtakes + /// a result held for a scheduled emit. + #[cfg(not(spawned_sim))] // native jobs are not simulated + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn a_deadline_overtakes_a_blocking_jobs_held_result() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe_blocking( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(400); + job.emit_at_blocking(at, Note("late".into())).unwrap(); + 7 + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 1).await, ["job:deadline"]); + // Past the emit's instant: it was dropped (the job is over). + spawned_rt::tasks::sleep(Duration::from_millis(500)).await; + assert_eq!(log.snapshot(), ["job:deadline"]); + actor.context().stop(); + actor.join().await; + } + + /// Review P1 (agy 5): a result held for scheduled emits keeps the job's + /// `MAX_OUTSTANDING` credit until its terminal message is enqueued. It + /// used to return the credit when the work returned, so held jobs (and + /// their scheduled emits) were not bounded by `MAX_OUTSTANDING`. + #[spawned_macros::sim_test] + async fn a_held_result_keeps_its_credit_until_the_terminal_message() { + let (actor, log) = start(); + let job = in_actor(&actor, |_, ctx| { + ctx.pipe( + JobOptions::new(), + |job| async move { + let at = Instant::now() + Duration::from_millis(200); + job.emit_at(at, Note("late".into())).await.unwrap(); + 1 + }, + |o| describe("job", o), + ) + .unwrap() + }) + .await; + // Handshake: the work has returned (its result is now held). + while job.is_running() { + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + } + assert!(log.snapshot().is_empty()); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 1, "a held result returned its credit early"); + assert_eq!(wait_for(&log, 2).await, ["late", "job:ok:1"]); + let outstanding = in_actor(&actor, |_, ctx| ctx.deferred_outstanding()).await; + assert_eq!(outstanding, 0); + actor.context().stop(); + actor.join().await; + } + + /// `emit_at` with an instant already reached (or in the past) is an + /// immediate emit: enqueued at the call, ahead of later emits. + #[spawned_macros::sim_test] + async fn emit_at_a_past_instant_is_delivered_immediately() { + let (actor, log) = start(); + in_actor(&actor, |_, ctx| { + ctx.pipe( + JobOptions::new(), + |job| async move { + let past = Instant::now(); + spawned_rt::tasks::sleep(Duration::from_millis(5)).await; + job.emit_at(past, Note("past".into())).await?; + job.emit(Note("later".into())).await + }, + |o| describe("job", o), + ) + .unwrap(); + }) + .await; + assert_eq!(wait_for(&log, 3).await, ["past", "later", "job:ok:Ok(())"]); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn send_handler_impls_answer_requests_with_unit() { + let (actor, log) = start(); + assert!(matches!(actor.request(Poke(false)).await, Ok(()))); + // The early return still answers. + assert!(matches!(actor.request(Poke(true)).await, Ok(()))); + actor.send(Poke(false)).unwrap(); + assert_eq!(wait_for(&log, 2).await, ["poked", "poked"]); + actor.context().stop(); + actor.join().await; + assert!(actor.request(Poke(false)).await.is_err()); + } + + #[spawned_macros::sim_test] + async fn child_spec_overrides_the_actor_bounds() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("bounds", 1).unwrap(); + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let log = Arc::new(Log::default()); + let child = builder + .add( + ChildSpec::new("bounded", &group, move |_| { + let log = log.clone(); + async move { Ok::<_, String>(Owner { log }) } + }) + .max_outstanding(2) + .max_native_jobs(1), + ) + .unwrap(); + let _supervisor = builder.start().await.unwrap(); + let actor = child.current().unwrap(); + let refused = in_actor(&actor, |_, ctx| { + let native = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("one native slot"); + let second_native = ctx.reserve_blocking_job(JobOptions::new()).err(); + drop(native); + let credits = ( + ctx.reserve_job(JobOptions::new()), + ctx.reserve_job(JobOptions::new()), + ); + assert!(credits.0.is_ok() && credits.1.is_ok(), "two credits"); + let second_credit = ctx.reserve_job(JobOptions::new()).err(); + drop(credits); + (second_native, second_credit) + }) + .await; + assert!(matches!(refused.0, Some(PipeError::NativeBudget))); + assert!(matches!(refused.1, Some(PipeError::Overloaded))); + let _ = system.shutdown(Duration::from_secs(5)).await; + } +} + +#[cfg(not(spawned_sim))] +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, Instant, JobGone, JobOptions, JobOutcome, PipeError, + }; + use spawned_concurrency::{actor, send_handler}; + use spawned_test::threads::in_actor; + use std::time::Duration; + + pub struct Owner { + log: Arc, + } + + #[actor(max_native_jobs = 1)] + impl Owner { + #[send_handler] + fn note(&mut self, msg: Note, _: &Context) { + self.log.push(msg.0); + } + } + + #[send_handler] + impl Handler for Owner { + fn handle(&mut self, msg: Poke, _ctx: &Context) { + if msg.0 { + return; + } + self.log.push("poked"); + } + } + + fn wait_for(log: &Log, n: usize) -> Vec { + for _ in 0..1000 { + let entries = log.snapshot(); + if entries.len() >= n { + return entries; + } + std::thread::sleep(Duration::from_millis(2)); + } + panic!("log never reached {n} entries: {:?}", log.snapshot()); + } + + fn label(name: &str, outcome: JobOutcome>) -> Note { + Note(format!("{name}:{:?}", outcome)) + } + + #[test] + fn in_actor_permits_and_emit_at() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + let refused = in_actor(&actor, |_, ctx| { + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("one native slot"); + // max_native_jobs = 1. + let second = ctx.reserve_blocking_job(JobOptions::new()).err(); + permit.pipe_blocking( + |job| { + let at = Instant::now() + Duration::from_millis(60); + job.emit_at_blocking(at, Note("late".into()))?; + job.emit_blocking(Note("now".into())) + }, + |o| label("job", o), + ); + second + }); + assert!(matches!(refused, Some(PipeError::NativeBudget))); + assert_eq!(wait_for(&log, 3), ["now", "late", "job:Ok(Ok(()))"]); + actor.context().stop(); + actor.join(); + } + + /// Review P1 (Codex 8, agy 1), threads mode: a hard deadline overtakes + /// a result held for a scheduled emit. + #[test] + fn a_deadline_overtakes_a_result_held_for_scheduled_emits() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + in_actor(&actor, |_, ctx| { + let opts = JobOptions::new().deadline(Duration::from_millis(40)); + ctx.pipe_blocking( + opts, + |job| { + let at = Instant::now() + Duration::from_millis(300); + job.emit_at_blocking(at, Note("late".into())) + }, + |o| label("job", o), + ) + .unwrap(); + }); + std::thread::sleep(Duration::from_millis(150)); + assert_eq!(log.snapshot(), ["job:Err(DeadlineExceeded)"]); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(log.snapshot(), ["job:Err(DeadlineExceeded)"]); + assert_eq!(in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), 0); + actor.context().stop(); + actor.join(); + } + + /// Review P1 (agy 5), threads mode: a held result keeps its credit. + #[test] + fn a_held_result_keeps_its_credit_until_the_terminal_message() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + let job = in_actor(&actor, |_, ctx| { + ctx.pipe_blocking( + JobOptions::new(), + |job| { + let at = Instant::now() + Duration::from_millis(200); + job.emit_at_blocking(at, Note("late".into())) + }, + |o| label("job", o), + ) + .unwrap() + }); + // Handshake: the work has returned (its result is now held). + while job.is_running() { + std::thread::sleep(Duration::from_millis(1)); + } + assert!(log.snapshot().is_empty()); + assert_eq!( + in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), + 1, + "a held result returned its credit early" + ); + assert_eq!(wait_for(&log, 2), ["late", "job:Ok(Ok(()))"]); + assert_eq!(in_actor(&actor, |_, ctx| ctx.deferred_outstanding()), 0); + actor.context().stop(); + actor.join(); + } + + /// `emit_at_blocking` with an instant already reached is immediate. + #[test] + fn emit_at_a_past_instant_is_delivered_immediately() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + in_actor(&actor, |_, ctx| { + ctx.pipe_blocking( + JobOptions::new(), + |job| { + let past = Instant::now(); + std::thread::sleep(Duration::from_millis(5)); + job.emit_at_blocking(past, Note("past".into()))?; + job.emit_blocking(Note("later".into())) + }, + |o| label("job", o), + ) + .unwrap(); + }); + assert_eq!(wait_for(&log, 3), ["past", "later", "job:Ok(Ok(()))"]); + actor.context().stop(); + actor.join(); + } + + #[test] + fn send_handler_impls_answer_requests_with_unit() { + let log = Arc::new(Log::default()); + let actor = Owner { log: log.clone() }.start(); + assert!(matches!(actor.request(Poke(false)), Ok(()))); + assert!(matches!(actor.request(Poke(true)), Ok(()))); + assert_eq!(log.snapshot(), ["poked"]); + actor.context().stop(); + actor.join(); + } +} + +/// Review P2 (Codex 14, agy 3): shortening a delegated token's deadline +/// wakes its watchers so they re-arm; the caller-gone watcher of a blocking +/// `pipe_reply` job must recheck the slot, not cancel the job. A blocking +/// linked job's deadline hands the token back while the work keeps running; +/// the handler then shortens the token and checks the job. +#[cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +mod caller_gone_vs_shortening_tasks { + use spawned_concurrency::message::Message; + use spawned_concurrency::tasks::{ + Actor, ActorStart, Context, Handler, JobError, JobHandle, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }; + use std::time::Duration; + + #[derive(Default)] + struct Delegator { + job: Option, + } + impl Actor for Delegator { + // Two blocking jobs at once (the default is 1 on one CPU). + const MAX_NATIVE_JOBS: Option = Some(2); + } + struct Ask; + impl Message for Ask { + type Result = (bool, bool); + } + struct Delegated(JobOutcome<()>, ReplyTo<(bool, bool)>); + impl Message for Delegated { + type Result = (); + } + impl Handler for Delegator { + async fn handle( + &mut self, + _: Ask, + reply: PendingReply<'_, (bool, bool)>, + ctx: &Context, + ) { + let token = reply.defer().unwrap(); + let opts = JobOptions::new() + .cancel_on_caller_gone() + .deadline(Duration::from_millis(20)); + let job = ctx + .pipe_reply_blocking( + opts, + token, + |_| std::thread::sleep(Duration::from_millis(300)), + Delegated, + ) + .unwrap(); + self.job = Some(job); + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + struct Probe; + impl Message for Probe { + type Result = bool; + } + struct Ended(ReplyTo<()>); + impl Message for Ended { + type Result = (); + } + impl Handler for Delegator { + async fn handle(&mut self, msg: Ended, _: PendingReply<'_, ()>, _: &Context) { + msg.0.reply(()); + } + } + impl Handler for Delegator { + async fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions::new().cancel_on_caller_gone(); + let job = ctx + .pipe_reply_blocking( + opts, + token, + |job| { + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(5)); + } + }, + |_, token| Ended(token), + ) + .unwrap(); + self.job = Some(job); + } + } + impl Handler for Delegator { + async fn handle(&mut self, _: Probe, reply: PendingReply<'_, bool>, _: &Context) { + reply.reply(self.job.as_ref().unwrap().is_cancelled()); + } + } + impl Handler for Delegator { + async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + let Delegated(outcome, token) = msg; + let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); + token.shorten_deadline(Duration::from_secs(30)); + let cancelled = self.job.as_ref().unwrap().is_cancelled(); + token.reply((deadline, cancelled)); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn shortening_the_deadline_does_not_cancel_a_caller_gone_job() { + let actor = Delegator::default().start(); + let (deadline, cancelled) = actor + .request_with_timeout(Ask, Duration::from_secs(60)) + .await + .unwrap(); + assert!(deadline, "the job deadline handed the token back"); + assert!(!cancelled, "a shortened deadline cancelled the job"); + // The caller leaving still cancels a blocking linked job. + let left = actor + .send_request(Leave, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert!( + !actor.request(Probe).await.unwrap(), + "cancelled before the caller left" + ); + drop(left); + let mut cancelled = false; + for _ in 0..500 { + cancelled = actor.request(Probe).await.unwrap(); + if cancelled { + break; + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + assert!(cancelled, "the caller left but the job was not cancelled"); + actor.context().stop(); + actor.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod caller_gone_vs_shortening_threads { + use spawned_concurrency::message::Message; + use spawned_concurrency::threads::{ + Actor, ActorStart, Context, Handler, JobError, JobHandle, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }; + use std::time::Duration; + + #[derive(Default)] + struct Delegator { + job: Option, + } + impl Actor for Delegator { + // Two blocking jobs at once (the default is 1 on one CPU). + const MAX_NATIVE_JOBS: Option = Some(2); + } + struct Ask; + impl Message for Ask { + type Result = (bool, bool); + } + struct Delegated(JobOutcome<()>, ReplyTo<(bool, bool)>); + impl Message for Delegated { + type Result = (); + } + impl Handler for Delegator { + fn handle(&mut self, _: Ask, reply: PendingReply<'_, (bool, bool)>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions::new() + .cancel_on_caller_gone() + .deadline(Duration::from_millis(20)); + let job = ctx + .pipe_reply_blocking( + opts, + token, + |_| std::thread::sleep(Duration::from_millis(300)), + Delegated, + ) + .unwrap(); + self.job = Some(job); + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + struct Probe; + impl Message for Probe { + type Result = bool; + } + struct Ended(ReplyTo<()>); + impl Message for Ended { + type Result = (); + } + impl Handler for Delegator { + fn handle(&mut self, msg: Ended, _: PendingReply<'_, ()>, _: &Context) { + msg.0.reply(()); + } + } + impl Handler for Delegator { + fn handle(&mut self, _: Leave, reply: PendingReply<'_, ()>, ctx: &Context) { + let token = reply.defer().unwrap(); + let opts = JobOptions::new().cancel_on_caller_gone(); + let job = ctx + .pipe_reply_blocking( + opts, + token, + |job| { + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(5)); + } + }, + |_, token| Ended(token), + ) + .unwrap(); + self.job = Some(job); + } + } + impl Handler for Delegator { + fn handle(&mut self, _: Probe, reply: PendingReply<'_, bool>, _: &Context) { + reply.reply(self.job.as_ref().unwrap().is_cancelled()); + } + } + impl Handler for Delegator { + fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + let Delegated(outcome, token) = msg; + let deadline = matches!(outcome, Err(JobError::DeadlineExceeded)); + token.shorten_deadline(Duration::from_secs(30)); + let cancelled = self.job.as_ref().unwrap().is_cancelled(); + token.reply((deadline, cancelled)); + } + } + + #[test] + fn shortening_the_deadline_does_not_cancel_a_caller_gone_job() { + let actor = Delegator::default().start(); + let (deadline, cancelled) = actor + .request_with_timeout(Ask, Duration::from_secs(60)) + .unwrap(); + assert!(deadline, "the job deadline handed the token back"); + assert!(!cancelled, "a shortened deadline cancelled the job"); + // The caller leaving still cancels a blocking linked job. + let left = actor + .send_request(Leave, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert!( + !actor.request(Probe).unwrap(), + "cancelled before the caller left" + ); + drop(left); + let mut cancelled = false; + for _ in 0..500 { + cancelled = actor.request(Probe).unwrap(); + if cancelled { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + assert!(cancelled, "the caller left but the job was not cancelled"); + actor.context().stop(); + actor.join(); + } +} diff --git a/concurrency/tests/deferred_jobs.rs b/concurrency/tests/deferred_jobs.rs new file mode 100644 index 000000000..9c493c5c0 --- /dev/null +++ b/concurrency/tests/deferred_jobs.rs @@ -0,0 +1,705 @@ +//! Pipe-to-self jobs (docs/design/deferred-replies.md ยง9): results as +//! messages, panics, keyed replacement and fencing, cancellation, job +//! deadlines, emits, delegation with `pipe_reply`, bounds and owner exit. +use spawned_concurrency::message::Message; +use std::sync::{Arc, Mutex}; + +/// What the owner's handlers observed, in dispatch order. +#[derive(Default)] +pub struct Log(pub Mutex>); +impl Log { + pub fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + pub fn take(&self) -> Vec { + std::mem::take(&mut self.0.lock().unwrap()) + } +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobGone, JobOptions, + JobOutcome, PendingReply, PipeError, PipeReplyError, ReplyTo, + }; + use spawned_concurrency::ActorError; + use spawned_rt::tasks::{oneshot, sleep}; + use std::time::Duration; + + struct Owner { + log: Arc, + gates: Vec>, + } + impl Actor for Owner { + const MAX_OUTSTANDING: usize = 4; + const MAX_NATIVE_JOBS: Option = Some(1); + async fn stopped(&mut self, _: &Context) { + self.log.push("stopped"); + } + } + + struct Done(String); + impl Message for Done { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } + } + + struct Open; + impl Message for Open { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Open, reply: PendingReply<'_, ()>, _: &Context) { + for gate in self.gates.drain(..) { + let _ = gate.send(()); + } + reply.reply(()); + } + } + + struct Outstanding; + impl Message for Outstanding { + type Result = usize; + } + impl Handler for Owner { + async fn handle( + &mut self, + _: Outstanding, + reply: PendingReply<'_, usize>, + ctx: &Context, + ) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn describe(name: &str, outcome: JobOutcome) -> Done { + Done(match outcome { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(JobError::Panicked(m)) => format!("{name}:panicked:{m}"), + Err(JobError::DeadlineExceeded) => format!("{name}:deadline"), + Err(other) => format!("{name}:{other}"), + }) + } + + /// Start jobs from inside a turn, as handlers do. + #[cfg_attr(spawned_sim, allow(dead_code))] // native jobs are not simulated + enum Start { + Pipe(&'static str, u64), + Panics, + Gated(&'static str, Option), + Keyed(&'static str), + CancelAfterSpawn, + Blocking(u32), + NativeBudget, + Emits(&'static str, u32), + } + impl Message for Start { + type Result = Result<(), String>; + } + impl Handler for Owner { + async fn handle( + &mut self, + msg: Start, + reply: PendingReply<'_, Result<(), String>>, + ctx: &Context, + ) { + let result = match msg { + Start::Pipe(name, v) => ctx + .pipe( + JobOptions::new(), + |_| async move { v * 2 }, + move |o| describe(name, o), + ) + .map(drop) + .map_err(|e| e.to_string()), + Start::Panics => ctx + .pipe( + JobOptions::new(), + |_| async { panic!("job boom") }, + |o: JobOutcome<()>| describe("p", o), + ) + .map(drop) + .map_err(|e| e.to_string()), + Start::Gated(name, deadline) => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + let opts = JobOptions::new() + .deadline(deadline.map_or(Deadline::Never, Deadline::after)); + ctx.pipe( + opts, + |_| async move { gate.await.is_ok() }, + move |o| describe(name, o), + ) + .map(drop) + .map_err(|e| e.to_string()) + } + Start::Keyed(name) => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + ctx.pipe( + JobOptions::new().key("k"), + |_| async move { gate.await.is_ok() }, + move |o| describe(name, o), + ) + .map(drop) + .map_err(|e| e.to_string()) + } + Start::CancelAfterSpawn => { + let (open, gate) = oneshot::channel::<()>(); + self.gates.push(open); + ctx.pipe( + JobOptions::new(), + |_| async move { gate.await.is_ok() }, + |o| describe("cancelled", o), + ) + .map(|job| job.cancel()) + .map_err(|e| e.to_string()) + } + Start::Blocking(v) => ctx + .pipe_blocking( + JobOptions::new(), + move |_| v + 1, + |o| describe("blocking", o), + ) + .map(drop) + .map_err(|e| e.to_string()), + Start::NativeBudget => { + let (hold, held) = std::sync::mpsc::channel::<()>(); + let first = ctx.pipe_blocking( + JobOptions::new(), + move |_| { + let _ = held.recv(); + }, + |o| describe("first", o), + ); + let second = + ctx.pipe_blocking(JobOptions::new(), |_| (), |o| describe("second", o)); + drop(hold); + match (&first, &second) { + (Ok(_), Err(PipeError::NativeBudget)) => Ok(()), + _ => Err(format!("{first:?} / {second:?}")), + } + } + Start::Emits(name, n) => ctx + .pipe_blocking( + JobOptions::new() + .key("emits") + .emit_credits(std::num::NonZeroU16::new(2).unwrap()), + move |job| { + for i in 0..n { + job.emit_blocking(Done(format!("{name}:emit:{i}")))?; + } + Ok::(n) + }, + move |o| describe(name, o), + ) + .map(drop) + .map_err(|e| e.to_string()), + }; + reply.reply(result); + } + } + + fn start() -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + let actor = Owner { + log: log.clone(), + gates: Vec::new(), + } + .start(); + (actor, log) + } + + /// Wait until the log has `n` entries (jobs report asynchronously). + async fn entries(actor: &ActorRef, log: &Log, n: usize) -> Vec { + for _ in 0..500 { + actor.request(Outstanding).await.unwrap(); + if log.0.lock().unwrap().len() >= n { + return log.take(); + } + sleep(Duration::from_millis(5)).await; + } + panic!("log never reached {n} entries: {:?}", log.take()); + } + + async fn stop(actor: ActorRef) { + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn job_results_and_panics_arrive_as_one_message() { + let (actor, log) = start(); + actor.request(Start::Pipe("a", 21)).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["a:ok:42"]); + actor.request(Start::Panics).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["p:panicked:job boom"]); + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + stop(actor).await; + } + + #[spawned_macros::sim_test] + async fn keyed_replacement_and_cancel_fence_results() { + let (actor, log) = start(); + actor.request(Start::Keyed("old")).await.unwrap().unwrap(); + actor.request(Start::Keyed("new")).await.unwrap().unwrap(); + actor + .request(Start::CancelAfterSpawn) + .await + .unwrap() + .unwrap(); + actor.request(Open).await.unwrap(); + // Only the replacing job reports; the replaced and cancelled ones are + // fenced even though their work finished. + assert_eq!(entries(&actor, &log, 1).await, ["new:ok:true"]); + sleep(Duration::from_millis(20)).await; + assert!(log.take().is_empty()); + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + stop(actor).await; + } + + /// Starts keyed job `old`, publishing its handle. + type OldJob = Arc)>>>; + struct StartOld(OldJob); + impl Message for StartOld { + type Result = (); + } + impl Handler for Owner { + async fn handle( + &mut self, + msg: StartOld, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { + let (open, gate) = oneshot::channel::<()>(); + let job = ctx + .pipe( + JobOptions::new().key("r"), + |_| async move { gate.await.map(|()| 1).unwrap_or(0) }, + |o| describe("old", o), + ) + .unwrap(); + *msg.0.lock().unwrap() = Some((job, open)); + reply.reply(()); + } + } + /// Holds its turn until `go` fires, then replaces the key. The turn + /// awaits instead of blocking the worker: a blocked worker strands the + /// old job, spawned into its non-stealable LIFO slot in the same poll, + /// so the old job never runs and the test hangs. + struct ReplaceWhen(oneshot::Receiver<()>); + impl Message for ReplaceWhen { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { + msg.0.await.unwrap(); + ctx.pipe( + JobOptions::new().key("r"), + |_| async { 2 }, + |o| describe("new", o), + ) + .unwrap(); + } + } + + /// Review P1 (`NoStaleKeyed`): the old keyed job finishes and queues its + /// terminal message while the owner is busy; the replacement made in that + /// busy turn fences it. + #[cfg(not(spawned_sim))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn replacement_fences_a_finished_keyed_job() { + let (actor, log) = start(); + let handle = Arc::new(Mutex::new(None)); + actor.request(StartOld(handle.clone())).await.unwrap(); + let (go, held) = oneshot::channel::<()>(); + actor.send(ReplaceWhen(held)).unwrap(); + // `ReplaceWhen` is queued before the old job can finish. + let (old, open) = handle.lock().unwrap().take().unwrap(); + open.send(()).unwrap(); + while old.is_running() { + sleep(Duration::from_millis(1)).await; + } + go.send(()).unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["new:ok:2"]); + sleep(Duration::from_millis(20)).await; + assert!( + log.take().is_empty(), + "the replaced job's terminal message was dispatched" + ); + stop(actor).await; + } + + struct Bomb; + impl Message for Bomb { + type Result = (); + } + impl Handler for Owner { + async fn handle(&mut self, _: Bomb, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.pipe( + JobOptions::new(), + |_| async {}, + |_: JobOutcome<()>| -> Done { panic!("done boom") }, + ) + .unwrap(); + reply.reply(()); + } + } + + /// Review P1: a panicking `done` is a handler panic: `stopped()` runs + /// and the owner's other jobs are cancelled. + #[spawned_macros::sim_test] + async fn a_panicking_done_runs_stopped_and_cancels_jobs() { + let (actor, log) = start(); + actor + .request(Start::Gated("survivor", None)) + .await + .unwrap() + .unwrap(); + actor.request(Bomb).await.unwrap(); + let reason = actor.wait_exit().await; + assert!( + matches!(reason, spawned_concurrency::ExitReason::Panic(_)), + "{reason:?}" + ); + assert!( + log.take().contains(&"stopped".to_string()), + "stopped() did not run" + ); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn job_deadline_notifies_while_the_work_keeps_its_credit() { + let (actor, log) = start(); + actor + .request(Start::Gated("slow", Some(Duration::from_millis(20)))) + .await + .unwrap() + .unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["slow:deadline"]); + assert_eq!( + actor.request(Outstanding).await.unwrap(), + 1, + "the hung work still holds its credit" + ); + actor.request(Open).await.unwrap(); + for _ in 0..200 { + if actor.request(Outstanding).await.unwrap() == 0 { + break; + } + sleep(Duration::from_millis(5)).await; + } + assert_eq!(actor.request(Outstanding).await.unwrap(), 0); + assert!(log.take().is_empty(), "the late result is dropped"); + stop(actor).await; + } + + #[spawned_macros::sim_test] + async fn pipe_rejects_at_the_credit_bound() { + let (actor, _) = start(); + for _ in 0..4 { + actor + .request(Start::Gated("g", None)) + .await + .unwrap() + .unwrap(); + } + let refused = actor.request(Start::Gated("g", None)).await.unwrap(); + assert!(refused.unwrap_err().contains("deferred credits")); + actor.request(Open).await.unwrap(); + stop(actor).await; + } + + #[cfg(not(spawned_sim))] + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn blocking_jobs_emit_in_order_and_respect_the_native_budget() { + let (actor, log) = start(); + actor.request(Start::Blocking(1)).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["blocking:ok:2"]); + actor.request(Start::NativeBudget).await.unwrap().unwrap(); + assert_eq!(entries(&actor, &log, 1).await, ["first:ok:()"]); + actor.request(Start::Emits("e", 5)).await.unwrap().unwrap(); + // Emits are FIFO and the terminal message is last. + assert_eq!( + entries(&actor, &log, 6).await, + [ + "e:emit:0", + "e:emit:1", + "e:emit:2", + "e:emit:3", + "e:emit:4", + "e:ok:Ok(5)" + ] + ); + stop(actor).await; + } + + // -- Delegation ----------------------------------------------------------- + + struct Backend; + impl Actor for Backend {} + struct Lookup(u64); + impl Message for Lookup { + type Result = u64; + } + impl Handler for Backend { + async fn handle(&mut self, msg: Lookup, reply: PendingReply<'_, u64>, _: &Context) { + sleep(Duration::from_millis(msg.0)).await; + reply.reply(msg.0 * 10); + } + } + + struct Router { + backend: ActorRef, + } + impl Actor for Router {} + struct Route(u64); + impl Message for Route { + type Result = u64; + } + struct Delegated { + res: JobOutcome>, + token: ReplyTo, + } + impl Message for Delegated { + type Result = (); + } + struct Ping; + impl Message for Ping { + type Result = &'static str; + } + struct Foreign(ReplyTo); + impl Message for Foreign { + type Result = bool; + } + impl Handler for Router { + async fn handle(&mut self, msg: Route, reply: PendingReply<'_, u64>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let downstream = self.backend.send_request(Lookup(msg.0), token.deadline()); + let work = async move { + match downstream { + Ok(rx) => rx.await, + Err(e) => Err(e), + } + }; + if let Err(e) = ctx.pipe_reply( + JobOptions::new(), + token, + |_| work, + |res, token| Delegated { res, token }, + ) { + drop(e.into_token()); + } + } + } + impl Handler for Router { + async fn handle(&mut self, msg: Delegated, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + Ok(Ok(v)) => { + msg.token.reply(v); + } + _ => msg.token.overloaded(), + } + } + } + impl Handler for Router { + async fn handle( + &mut self, + _: Ping, + reply: PendingReply<'_, &'static str>, + _: &Context, + ) { + reply.reply("pong"); + } + } + impl Handler for Router { + async fn handle( + &mut self, + msg: Foreign, + reply: PendingReply<'_, bool>, + ctx: &Context, + ) { + let refused = ctx.pipe_reply( + JobOptions::new(), + msg.0, + |_| async { 1u64 }, + |_, token| Delegated { + res: Ok(Ok(0)), + token, + }, + ); + let wrong_owner = matches!(refused, Err(PipeReplyError::WrongOwner(_))); + if let Err(e) = refused { + e.into_token().reply(7); + } + reply.reply(wrong_owner); + } + } + + /// Holds a deferred token so a test can hand it to another actor. + struct Holder(Option>); + impl Actor for Holder {} + struct Hold; + impl Message for Hold { + type Result = u64; + } + struct Give; + impl Message for Give { + type Result = Option>; + } + impl Handler for Holder { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u64>, _: &Context) { + self.0 = reply.defer().ok(); + } + } + impl Handler for Holder { + async fn handle( + &mut self, + _: Give, + reply: PendingReply<'_, Option>>, + _: &Context, + ) { + let token = self.0.take(); + reply.reply(token); + } + } + + #[spawned_macros::sim_test] + async fn router_delegates_with_pipe_reply_while_serving_its_mailbox() { + let backend = Backend.start(); + let router = Router { + backend: backend.clone(), + } + .start(); + let slow = router + .send_request(Route(200), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + // The router's mailbox is free while the slow request is delegated. + assert_eq!(router.request(Ping).await.unwrap(), "pong"); + assert_eq!(router.request(Route(1)).await.unwrap(), 10); + assert_eq!(slow.await.unwrap(), 2000); + router.context().stop(); + router.join().await; + backend.context().stop(); + backend.join().await; + } + + #[spawned_macros::sim_test] + async fn pipe_reply_refuses_a_token_of_another_actor() { + let backend = Backend.start(); + let router = Router { + backend: backend.clone(), + } + .start(); + let holder = Holder(None).start(); + let rx = holder.send_request(Hold, Deadline::Never).unwrap(); + let token = holder.request(Give).await.unwrap().unwrap(); + assert!(router.request(Foreign(token)).await.unwrap()); + assert_eq!( + rx.await.unwrap(), + 7, + "the refused token came back and was answered" + ); + for actor in [ + router.child_handle(), + backend.child_handle(), + holder.child_handle(), + ] { + actor.stop(); + } + router.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobOptions, JobOutcome, + PendingReply, + }; + + struct Owner { + log: Arc, + } + impl Actor for Owner {} + + struct Done(String); + impl Message for Done { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } + } + struct Compute(u64); + impl Message for Compute { + type Result = u64; + } + struct Computed { + res: JobOutcome, + token: spawned_concurrency::ReplyTo, + } + impl Message for Computed { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, msg: Compute, reply: PendingReply<'_, u64>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let n = msg.0; + let _ = ctx.pipe_reply_blocking( + JobOptions::new(), + token, + move |job| { + job.emit_blocking(Done(format!("computing {n}"))).unwrap(); + if n == 0 { + panic!("zero"); + } + n * n + }, + |res, token| Computed { res, token }, + ); + } + } + impl Handler for Owner { + fn handle(&mut self, msg: Computed, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + Ok(v) => { + msg.token.reply(v); + } + Err(JobError::Panicked(m)) => { + self.log.push(format!("panicked {m}")); + msg.token.overloaded(); + } + Err(_) => msg.token.overloaded(), + } + } + } + + #[test] + fn native_jobs_answer_deferred_requests() { + let log = Arc::new(Log::default()); + let actor: ActorRef = Owner { log: log.clone() }.start(); + let rx = actor.send_request(Compute(7), Deadline::Never).unwrap(); + assert_eq!(rx.recv().unwrap(), 49); + assert!(matches!( + actor.request(Compute(0)), + Err(spawned_concurrency::ActorError::Overloaded) + )); + let entries = log.take(); + assert_eq!(entries[0], "computing 7"); + assert!( + entries.contains(&"panicked zero".to_string()), + "{entries:?}" + ); + actor.context().stop(); + actor.join(); + } +} diff --git a/concurrency/tests/deferred_replies.rs b/concurrency/tests/deferred_replies.rs new file mode 100644 index 000000000..0c9cfbea6 --- /dev/null +++ b/concurrency/tests/deferred_replies.rs @@ -0,0 +1,719 @@ +//! Deferred replies (docs/design/deferred-replies.md): every discharge path of +//! a request's reply slot, in `tasks` mode (also under `spawned_sim`) and in +//! `threads` mode. +use spawned_concurrency::{ + message::Message, ActorError, Deadline, DeferError, Delivery, ExitReason, ReplyTo, +}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +/// Park the request: the handler defers it and stores the token. +pub struct Park(pub u64); +impl Message for Park { + type Result = u64; +} +/// Answer every parked request with its value plus `0`. +pub struct Release(pub u64); +impl Message for Release { + type Result = usize; +} +/// Drop every parked token between turns. +pub struct DropAll; +impl Message for DropAll { + type Result = (); +} +/// Return without answering (the `PendingReply` is dropped in the turn). +pub struct Forget; +impl Message for Forget { + type Result = u64; +} +/// Panic in the turn, after optionally deferring into a local. +pub struct Explode { + pub defer_first: bool, +} +impl Message for Explode { + type Result = u64; +} +/// Report what `defer()` returned (for sends, expired requests). +pub struct TryDefer; +impl Message for TryDefer { + type Result = (); +} +/// Stop, answering parked tokens in `stopped()` when `answer` is set. +pub struct Stop { + pub answer: bool, +} +impl Message for Stop { + type Result = (); +} +pub struct Outstanding; +impl Message for Outstanding { + type Result = usize; +} + +/// Shared observations of the deferred-reply actor. +#[derive(Default)] +pub struct Seen { + pub defer_errors: Mutex>, + pub late: Mutex>, +} + +#[cfg(feature = "async")] +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + use spawned_rt::tasks::sleep; + + pub struct Parker { + pub parked: Vec<(u64, ReplyTo)>, + pub answer_in_stopped: bool, + pub seen: Arc, + } + impl Parker { + pub fn new(seen: Arc) -> Self { + Self { + parked: Vec::new(), + answer_in_stopped: false, + seen, + } + } + } + impl Actor for Parker { + const MAX_OUTSTANDING: usize = 2; + async fn stopped(&mut self, _ctx: &Context) { + if self.answer_in_stopped { + for (v, token) in self.parked.drain(..) { + token.reply(v + 1000); + } + } + } + } + impl Handler for Parker { + async fn handle(&mut self, msg: Park, reply: PendingReply<'_, u64>, _: &Context) { + match reply.defer() { + Ok(token) => self.parked.push((msg.0, token)), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + async fn handle( + &mut self, + msg: Release, + reply: PendingReply<'_, usize>, + _: &Context, + ) { + let n = self.parked.len(); + for (v, token) in self.parked.drain(..) { + let delivery = token.reply(v + msg.0); + if delivery != Delivery::Delivered { + self.seen.late.lock().unwrap().push(delivery); + } + } + reply.reply(n); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: DropAll, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.clear(); + reply.reply(()); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: Forget, _reply: PendingReply<'_, u64>, _: &Context) {} + } + impl Handler for Parker { + async fn handle(&mut self, msg: Explode, reply: PendingReply<'_, u64>, _: &Context) { + let _held = if msg.defer_first { + Some(reply.defer().expect("defer")) + } else { + None + }; + panic!("explode"); + } + } + impl Handler for Parker { + async fn handle(&mut self, _: TryDefer, reply: PendingReply<'_, ()>, _: &Context) { + match reply.defer() { + Ok(token) => drop(token), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + async fn handle(&mut self, msg: Stop, reply: PendingReply<'_, ()>, ctx: &Context) { + self.answer_in_stopped = msg.answer; + ctx.stop(); + reply.reply(()); + } + } + impl Handler for Parker { + async fn handle( + &mut self, + _: Outstanding, + reply: PendingReply<'_, usize>, + ctx: &Context, + ) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn start() -> (spawned_concurrency::tasks::ActorRef, Arc) { + let seen = Arc::new(Seen::default()); + (Parker::new(seen.clone()).start(), seen) + } + + async fn settle(actor: &spawned_concurrency::tasks::ActorRef) -> usize { + actor.request(Outstanding).await.unwrap() + } + + #[spawned_macros::sim_test] + async fn deferred_reply_answers_from_a_later_turn() { + let (actor, _) = start(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); + let b = actor.send_request(Park(2), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 2); + // The mailbox kept serving while both requests were parked. + assert_eq!(actor.request(Release(10)).await.unwrap(), 2); + assert_eq!(a.await.unwrap(), 11); + assert_eq!(b.await.unwrap(), 12); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn dropped_token_between_turns_answers_reply_dropped() { + let (actor, _) = start(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); + actor.request(DropAll).await.unwrap(); + assert!(matches!(rx.await, Err(ActorError::ReplyDropped))); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn unanswered_turn_answers_reply_dropped() { + let (actor, _) = start(); + assert!(matches!( + actor.request(Forget).await, + Err(ActorError::ReplyDropped) + )); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn panicking_turn_answers_actor_stopped() { + for defer_first in [false, true] { + let (actor, _) = start(); + let result = actor.request(Explode { defer_first }).await; + // v0.8: answered at the death point with the final reason. + assert!( + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), + "{result:?}" + ); + actor.join().await; + } + } + + #[spawned_macros::sim_test] + async fn defer_at_the_bound_answers_overloaded() { + let (actor, seen) = start(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); + let b = actor.send_request(Park(2), Deadline::Never).unwrap(); + // MAX_OUTSTANDING is 2: the third deferral is refused in the same step. + assert!(matches!( + actor.request(Park(3)).await, + Err(ActorError::Overloaded) + )); + // The caller is answered in the deferring step; the handler's record + // of it is visible once a later turn ran (FIFO). + settle(&actor).await; + assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); + assert_eq!(actor.request(Release(0)).await.unwrap(), 2); + assert_eq!((a.await.unwrap(), b.await.unwrap()), (1, 2)); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn abandoned_caller_returns_the_credit_and_drops_the_late_reply() { + let (actor, seen) = start(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 1); + drop(rx); + assert_eq!( + settle(&actor).await, + 0, + "abandonment returns the credit at once" + ); + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::CallerGone]); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn sends_and_expired_requests_cannot_defer() { + let (actor, seen) = start(); + actor.send(TryDefer).unwrap(); + let expired = actor + .send_request(TryDefer, Deadline::after(Duration::ZERO)) + .unwrap(); + assert!(matches!(expired.await, Err(ActorError::RequestTimeout))); + settle(&actor).await; + assert_eq!( + *seen.defer_errors.lock().unwrap(), + [DeferError::NoCaller, DeferError::Closed] + ); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn late_reply_after_the_deadline_is_expired() { + let (actor, seen) = start(); + let rx = actor + .send_request(Park(1), Deadline::after(Duration::from_millis(50))) + .unwrap(); + assert!(matches!(rx.await, Err(ActorError::RequestTimeout))); + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert_eq!(settle(&actor).await, 0); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn unpolled_deferred_token_expires_by_the_owner_heap() { + let (actor, seen) = start(); + // Nobody polls this receiver: only the owner's deadline heap can + // expire the token and return its credit. + let rx = actor + .send_request(Park(1), Deadline::after(Duration::from_millis(20))) + .unwrap(); + assert_eq!(settle(&actor).await, 1); + let mut waited = Duration::ZERO; + while settle(&actor).await != 0 { + assert!(waited < Duration::from_secs(5), "token never expired"); + spawned_rt::tasks::sleep(Duration::from_millis(10)).await; + waited += Duration::from_millis(10); + } + actor.request(Release(0)).await.unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert!(matches!(rx.await, Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join().await; + } + + /// Review P1: destroying the runtime under a live actor still closes its + /// registry; a `Deadline::Never` caller is answered `ActorStopped`. + #[cfg(not(spawned_sim))] + #[test] + fn runtime_destruction_closes_the_registry() { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .unwrap(); + let (actor, rx) = rt.block_on(async { + let (actor, _) = start(); + let rx = actor.send_request(Park(1), Deadline::Never).unwrap(); + assert_eq!(settle(&actor).await, 1); + (actor, rx) + }); + drop(rt); + let caller = std::thread::spawn(move || rx.recv()); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while !caller.is_finished() { + assert!(std::time::Instant::now() < deadline, "the caller hangs"); + std::thread::sleep(Duration::from_millis(5)); + } + let result = caller.join().unwrap(); + // A runtime-dropped actor ends Killed (G1); its callers see it. + assert!( + matches!(result, Err(ActorError::ActorStopped(ExitReason::Killed))), + "{result:?}" + ); + drop(actor); + } + + #[spawned_macros::sim_test] + async fn stopped_may_answer_stored_tokens() { + let (actor, _) = start(); + let rx = actor.send_request(Park(7), Deadline::Never).unwrap(); + actor.request(Stop { answer: true }).await.unwrap(); + assert_eq!(rx.await.unwrap(), 1007); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn death_answers_stored_tokens_actor_stopped() { + let (actor, _) = start(); + let rx = actor.send_request(Park(7), Deadline::Never).unwrap(); + let queued_behind_stop = { + actor.request(Stop { answer: false }).await.unwrap(); + actor.send_request(Park(8), Deadline::Never) + }; + assert!(matches!( + rx.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + // Sent while exiting it is admitted and answered at D; sent after D + // it reports NoProc. + match queued_behind_stop { + Ok(queued) => assert!(matches!( + queued.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )), + Err(error) => assert!(matches!(error, ActorError::NoProc), "{error:?}"), + } + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn token_closed_resolves_when_the_caller_leaves() { + use spawned_concurrency::tasks::ActorRef; + struct Watcher { + closed: Option>, + } + impl Actor for Watcher {} + struct Watch; + impl Message for Watch { + type Result = (); + } + struct Ping; + impl Message for Ping { + type Result = (); + } + struct Done; + impl Message for Done { + type Result = bool; + } + impl Handler for Watcher { + async fn handle(&mut self, _: Watch, reply: PendingReply<'_, ()>, _: &Context) { + let token = reply.defer().unwrap(); + self.closed = Some(spawned_rt::tasks::spawn(async move { + token.closed().await; + token.is_closed() + })); + } + } + impl Handler for Watcher { + async fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } + } + impl Handler for Watcher { + async fn handle(&mut self, _: Done, reply: PendingReply<'_, bool>, _: &Context) { + let closed = self.closed.take().unwrap().await.unwrap(); + reply.reply(closed); + } + } + let actor: ActorRef = Watcher { closed: None }.start(); + let rx = actor.send_request(Watch, Deadline::Never).unwrap(); + actor.request(Ping).await.unwrap(); // `Watch` was handled (FIFO) + drop(rx); + assert!(actor.request(Done).await.unwrap()); + actor.context().stop(); + actor.join().await; + } + + /// agy #11: `closed()` re-arms when the deadline is shortened while it + /// waits. + #[spawned_macros::sim_test] + async fn closed_follows_a_shortened_deadline() { + let (reply, _rx) = spawned_concurrency::__private::reply_pair::(Deadline::Never); + let token = std::sync::Arc::new(reply.defer().unwrap()); + let watcher = { + let token = token.clone(); + spawned_rt::tasks::spawn(async move { token.closed().await }) + }; + sleep(Duration::from_millis(5)).await; + token.shorten_deadline(Duration::from_millis(20)); + spawned_rt::tasks::timeout(Duration::from_secs(5), watcher) + .await + .expect("closed() missed the shortened deadline") + .unwrap(); + assert!(token.is_closed()); + } + + /// Review P1 (agy 4): shortening a token's deadline wakes an awaiting + /// caller, so its own timer re-arms for the earlier deadline. Without + /// the wake it kept sleeping until the deadline fixed at send; real time + /// was saved by the owner's heap, but a paused Tokio clock auto-advanced + /// to the old deadline (here 60 s instead of 50 ms). + #[cfg(not(spawned_sim))] + #[tokio::test(start_paused = true)] + async fn a_shortened_deadline_rearms_the_callers_timer() { + struct Shortener(Option>); + impl Actor for Shortener {} + struct Shorten; + impl spawned_concurrency::message::Message for Shorten { + type Result = (); + } + impl Handler for Shortener { + async fn handle(&mut self, _: Shorten, reply: PendingReply<'_, ()>, _: &Context) { + let token = reply.defer().unwrap(); + token.shorten_deadline(Duration::from_millis(50)); + self.0 = Some(token); + } + } + let actor = Shortener(None).start(); + let began = tokio::time::Instant::now(); + let result = actor + .request_with_timeout(Shorten, Duration::from_secs(60)) + .await; + assert!( + matches!(result, Err(ActorError::RequestTimeout)), + "{result:?}" + ); + assert!( + began.elapsed() < Duration::from_secs(5), + "the caller slept until the old deadline: {:?}", + began.elapsed() + ); + actor.context().stop(); + actor.join().await; + } + + /// `ReplyDropped` (the handler chose not to answer) is not transient; + /// `overloaded()`, a handler's "not now, try again", is. + #[spawned_macros::sim_test] + async fn a_dropped_reply_is_not_transient_but_overloaded_is() { + struct Picky; + impl Actor for Picky {} + struct Ask(bool); + impl Message for Ask { + type Result = (); + } + impl Handler for Picky { + async fn handle(&mut self, msg: Ask, reply: PendingReply<'_, ()>, _: &Context) { + if msg.0 { + reply.overloaded(); + } // otherwise dropped unanswered + } + } + let actor = Picky.start(); + let dropped = actor.request(Ask(false)).await.unwrap_err(); + assert!(matches!(dropped, ActorError::ReplyDropped)); + assert!(!dropped.is_transient() && !dropped.target_gone()); + let busy = actor.request(Ask(true)).await.unwrap_err(); + assert!(matches!(busy, ActorError::Overloaded)); + assert!(busy.is_transient()); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn request_timeout_starts_at_send() { + let (actor, _) = start(); + // Sent now with a short deadline, polled only after it passed. + let response = actor + .send_request(Park(1), Deadline::after(Duration::from_millis(10))) + .unwrap(); + spawned_rt::tasks::sleep(Duration::from_millis(50)).await; + assert!(matches!(response.await, Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join().await; + } + + #[spawned_macros::sim_test] + async fn reply_pair_drives_a_manual_handler_directly() { + let seen = Arc::new(Seen::default()); + let mut parker = Parker::new(seen); + let actor = Parker::new(Arc::new(Seen::default())).start(); + let ctx = actor.context(); + let (reply, rx) = spawned_concurrency::__private::reply_pair::(Deadline::Never); + parker.handle(Park(5), reply, &ctx).await; + let (release, released) = + spawned_concurrency::__private::reply_pair::(Deadline::Never); + parker.handle(Release(1), release, &ctx).await; + assert_eq!(released.await.unwrap(), 1); + assert_eq!(rx.await.unwrap(), 6); + actor.context().stop(); + actor.join().await; + } +} + +#[cfg(not(spawned_sim))] +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::PendingReply; + + pub struct Parker { + parked: Vec<(u64, ReplyTo)>, + answer_in_stopped: bool, + seen: Arc, + } + impl Actor for Parker { + const MAX_OUTSTANDING: usize = 2; + fn stopped(&mut self, _ctx: &Context) { + if self.answer_in_stopped { + for (v, token) in self.parked.drain(..) { + token.reply(v + 1000); + } + } + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Park, reply: PendingReply<'_, u64>, _: &Context) { + match reply.defer() { + Ok(token) => self.parked.push((msg.0, token)), + Err(e) => self.seen.defer_errors.lock().unwrap().push(e), + } + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Release, reply: PendingReply<'_, usize>, _: &Context) { + let n = self.parked.len(); + for (v, token) in self.parked.drain(..) { + let delivery = token.reply(v + msg.0); + if delivery != Delivery::Delivered { + self.seen.late.lock().unwrap().push(delivery); + } + } + reply.reply(n); + } + } + impl Handler for Parker { + fn handle(&mut self, _: DropAll, reply: PendingReply<'_, ()>, _: &Context) { + self.parked.clear(); + reply.reply(()); + } + } + impl Handler for Parker { + fn handle(&mut self, _: Forget, _reply: PendingReply<'_, u64>, _: &Context) {} + } + impl Handler for Parker { + fn handle(&mut self, msg: Explode, reply: PendingReply<'_, u64>, _: &Context) { + let _held = if msg.defer_first { + Some(reply.defer().expect("defer")) + } else { + None + }; + panic!("explode"); + } + } + impl Handler for Parker { + fn handle(&mut self, msg: Stop, reply: PendingReply<'_, ()>, ctx: &Context) { + self.answer_in_stopped = msg.answer; + ctx.stop(); + reply.reply(()); + } + } + impl Handler for Parker { + fn handle(&mut self, _: Outstanding, reply: PendingReply<'_, usize>, ctx: &Context) { + reply.reply(ctx.deferred_outstanding()); + } + } + + fn start() -> (spawned_concurrency::threads::ActorRef, Arc) { + let seen = Arc::new(Seen::default()); + let actor = Parker { + parked: Vec::new(), + answer_in_stopped: false, + seen: seen.clone(), + } + .start(); + (actor, seen) + } + + #[test] + fn every_discharge_path() { + // Later turn. + let (actor, seen) = start(); + let a = actor.send_request(Park(1), Deadline::Never).unwrap(); + assert_eq!(actor.request(Outstanding).unwrap(), 1); + assert_eq!(actor.request(Release(10)).unwrap(), 1); + assert_eq!(a.recv().unwrap(), 11); + // Dropped between turns. + let b = actor.send_request(Park(1), Deadline::Never).unwrap(); + actor.request(DropAll).unwrap(); + assert!(matches!(b.recv(), Err(ActorError::ReplyDropped))); + // Unanswered turn. + assert!(matches!( + actor.request(Forget), + Err(ActorError::ReplyDropped) + )); + // Bound. + let c = actor.send_request(Park(1), Deadline::Never).unwrap(); + let d = actor.send_request(Park(2), Deadline::Never).unwrap(); + assert!(matches!( + actor.request(Park(3)), + Err(ActorError::Overloaded) + )); + // Abandonment returns the credit. + drop(c); + assert_eq!(actor.request(Outstanding).unwrap(), 1); + // The handler's record is visible after a later turn (FIFO). + assert_eq!(*seen.defer_errors.lock().unwrap(), [DeferError::Overloaded]); + // Expired deadline: the receiver expires the slot; the late reply is dropped. + let e = actor + .send_request(Park(4), Deadline::after(Duration::from_millis(20))) + .unwrap(); + assert!(matches!(e.recv(), Err(ActorError::RequestTimeout))); + actor.request(Release(0)).unwrap(); + assert_eq!(d.recv().unwrap(), 2); + assert_eq!( + *seen.late.lock().unwrap(), + [Delivery::CallerGone, Delivery::Expired] + ); + assert_eq!(actor.request(Outstanding).unwrap(), 0); + // stopped() may answer; then death closes the rest. + let f = actor.send_request(Park(7), Deadline::Never).unwrap(); + actor.request(Stop { answer: true }).unwrap(); + assert_eq!(f.recv().unwrap(), 1007); + actor.join(); + // Past the death point: NoProc (v0.8, decision 4). + assert!(matches!( + actor.request(Outstanding), + Err(ActorError::NoProc) + )); + } + + #[test] + fn unpolled_deferred_token_expires_by_the_owner_heap() { + let (actor, seen) = start(); + let rx = actor + .send_request(Park(1), Deadline::after(Duration::from_millis(20))) + .unwrap(); + let began = std::time::Instant::now(); + while actor.request(Outstanding).unwrap() != 0 { + assert!( + began.elapsed() < Duration::from_secs(5), + "token never expired" + ); + std::thread::sleep(Duration::from_millis(5)); + } + actor.request(Release(0)).unwrap(); + assert_eq!(*seen.late.lock().unwrap(), [Delivery::Expired]); + assert!(matches!(rx.recv(), Err(ActorError::RequestTimeout))); + actor.context().stop(); + actor.join(); + } + + #[test] + fn panic_and_death_answer_actor_stopped() { + for defer_first in [false, true] { + let (actor, _) = start(); + assert!(matches!( + actor.request(Explode { defer_first }), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + actor.join(); + } + let (actor, _) = start(); + let parked = actor.send_request(Park(1), Deadline::Never).unwrap(); + actor.request(Stop { answer: false }).unwrap(); + assert!(matches!( + parked.recv(), + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + actor.join(); + } +} diff --git a/concurrency/tests/deferred_sim.rs b/concurrency/tests/deferred_sim.rs new file mode 100644 index 000000000..c3b19d472 --- /dev/null +++ b/concurrency/tests/deferred_sim.rs @@ -0,0 +1,339 @@ +//! Simulator races for deferred replies and pipe-to-self jobs +//! (docs/design/deferred-replies.md), explored with PCT and bounded DFS over +//! real actors: a stored token's reply against its deadline, the owner's +//! death and the caller leaving; keyed replacement against the old job's +//! completion; emits against the terminal message and a cancel. Run with: +//! +//! ```sh +//! RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" cargo test -p spawned-concurrency --test deferred_sim +//! ``` +#![cfg(all(spawned_sim, feature = "async"))] + +use spawned_concurrency::{ + message::Message, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Delivery, Handler, JobCtx, JobHandle, + JobOptions, JobOutcome, PendingReply, ReplyTo, + }, + ActorError, +}; +use spawned_rt::tasks::{ + sim::explore::{explore, Strategy}, + sleep, spawn, +}; +use std::{ + future::Future, + sync::{Arc, Mutex}, + time::Duration, +}; + +type Log = Arc>>; + +fn push(log: &Log, entry: impl Into) { + log.lock().unwrap().push(entry.into()); +} + +fn entries(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +struct Owner { + log: Log, + stored: Vec>, + job: Option, +} +impl Actor for Owner {} + +fn owner(log: &Log) -> ActorRef { + Owner { + log: log.clone(), + stored: Vec::new(), + job: None, + } + .start() +} + +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +/// Answer every stored token and log how each delivery went. +struct Flush; +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Flush, _: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + let delivery = token.reply(7); + push(&self.log, format!("{delivery:?}")); + } + } +} + +struct StartKeyed(&'static str, u64); +impl Message for StartKeyed { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: StartKeyed, _: PendingReply<'_, ()>, ctx: &Context) { + let StartKeyed(name, delay) = msg; + ctx.pipe( + JobOptions::new().key("k"), + |_| async move { + sleep(Duration::from_millis(delay)).await; + name + }, + |outcome: JobOutcome<&'static str>| Done(outcome.unwrap()), + ) + .unwrap(); + push(&self.log, format!("start:{name}")); + } +} + +struct Done(&'static str); +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("done:{}", msg.0)); + } +} + +struct Tick(u32); +impl Message for Tick { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Tick, _: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("tick:{}", msg.0)); + } +} + +struct StartEmitter; +impl Message for StartEmitter { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: StartEmitter, _: PendingReply<'_, ()>, ctx: &Context) { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + let job = ctx + .pipe( + opts, + |job: JobCtx| async move { + for i in 0..3 { + if job.emit(Tick(i)).await.is_err() { + break; + } + sleep(Duration::from_millis(1)).await; + } + "emitter" + }, + |outcome: JobOutcome<&'static str>| Done(outcome.unwrap()), + ) + .unwrap(); + self.job = Some(job); + } +} + +struct CancelJob; +impl Message for CancelJob { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: CancelJob, _: PendingReply<'_, ()>, _: &Context) { + if let Some(job) = self.job.take() { + job.cancel(); + push(&self.log, "cancelled"); + } + } +} + +// --------------------------------------------------------------------------- +// Bodies +// --------------------------------------------------------------------------- + +/// A stored token's reply races its deadline, the owner's stop and (on odd +/// seeds) the caller dropping its receiver. The caller sees `Ok` exactly when +/// the reply reports `Delivered`, and always gets an answer. +async fn reply_vs_expiry_vs_death_vs_drop(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + let d = Duration::from_millis(10); + let receiver = actor + .send_request(Hold, Deadline::At(spawned_rt::time::Instant::now() + d)) + .unwrap(); + actor.request(Sync).await.unwrap(); + let replier = actor.clone(); + spawn(async move { + sleep(d).await; + let _ = replier.send(Flush); + }); + let stopper = actor.clone(); + spawn(async move { + sleep(d).await; + stopper.context().stop(); + }); + let awaited = seed % 2 == 0; + let caller = spawn(async move { + if awaited { + Some(receiver.await) + } else { + sleep(d).await; + drop(receiver); + None + } + }); + let result = caller.await.unwrap(); + actor.join().await; + let deliveries = entries(&log); + assert!(deliveries.len() <= 1, "{deliveries:?}"); + let delivered = deliveries.first().map(String::as_str) == Some("Delivered"); + match result { + Some(Ok(7)) => assert!(delivered, "Ok without a delivery: {deliveries:?}"), + Some(Ok(other)) => panic!("unexpected value {other}"), + Some(Err(e)) => { + assert!(!delivered, "{e:?} after a delivery"); + assert!( + matches!(e, ActorError::RequestTimeout | ActorError::ActorStopped(_)), + "{e:?}" + ); + } + None => {} + } + if let Some(delivery) = deliveries.first() { + let expected = [ + format!("{:?}", Delivery::Delivered), + format!("{:?}", Delivery::Expired), + format!("{:?}", Delivery::CallerGone), + ]; + assert!(expected.contains(delivery), "{delivery}"); + } +} + +/// A keyed job is replaced while it completes: its result never reaches a +/// turn after the replacement started. +async fn keyed_replacement(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + actor.send(StartKeyed("a", 5)).unwrap(); + let replacer = actor.clone(); + spawn(async move { + sleep(Duration::from_millis(4 + seed % 3)).await; + let _ = replacer.send(StartKeyed("b", 1)); + }); + for _ in 0..100 { + if entries(&log).iter().any(|e| e == "done:b") { + break; + } + sleep(Duration::from_millis(1)).await; + } + actor.request(Sync).await.unwrap(); + actor.context().stop(); + actor.join().await; + let log = entries(&log); + assert!(log.contains(&"done:b".to_string()), "{log:?}"); + let replaced = log.iter().position(|e| e == "start:b").unwrap(); + assert!( + !log[replaced..].contains(&"done:a".to_string()), + "a replaced job's result after its replacement: {log:?}" + ); +} + +/// Emits race the terminal message and a cancel: no tick after the terminal +/// message, and nothing from the job after the cancelling turn. +async fn emit_vs_terminal_and_cancel(seed: u64) { + let log = Log::default(); + let actor = owner(&log); + actor.send(StartEmitter).unwrap(); + let canceller = actor.clone(); + spawn(async move { + sleep(Duration::from_millis(seed % 5)).await; + let _ = canceller.send(CancelJob); + }); + sleep(Duration::from_millis(10)).await; + actor.request(Sync).await.unwrap(); + actor.context().stop(); + actor.join().await; + let log = entries(&log); + if let Some(done) = log.iter().position(|e| e.starts_with("done:")) { + assert!( + !log[done + 1..].iter().any(|e| e.starts_with("tick:")), + "an emit after the terminal message: {log:?}" + ); + } + if let Some(cancelled) = log.iter().position(|e| e == "cancelled") { + assert!( + log[cancelled + 1..].is_empty(), + "a cancelled job's message dispatched: {log:?}" + ); + } + let ticks: Vec<_> = log.iter().filter(|e| e.starts_with("tick:")).collect(); + let in_order = ticks + .iter() + .enumerate() + .all(|(i, t)| **t == format!("tick:{i}")); + assert!(in_order, "emits out of order: {log:?}"); +} + +// --------------------------------------------------------------------------- +// Driver +// --------------------------------------------------------------------------- + +const STRATEGIES: [(&str, usize); 2] = [("pct:d=3", 300), ("dfs:bound=3,depth=300", 300)]; + +fn check(name: &str, body: F) +where + F: Fn(u64) -> Fut + std::marker::Sync, + Fut: Future, +{ + for (strategy, budget) in STRATEGIES { + let report = explore(&Strategy::parse(strategy).unwrap(), 0, budget, &body); + assert!( + report.found.is_none(), + "{name} under {strategy}: {:?}", + report.found + ); + println!( + "{name} {strategy}: {} runs, {} schedules", + report.runs, report.schedules + ); + } +} + +#[test] +fn reply_races_expiry_death_and_caller_drop() { + check( + "reply vs expiry vs death vs drop", + reply_vs_expiry_vs_death_vs_drop, + ); +} + +#[test] +fn keyed_replacement_fences_the_completing_job() { + check("keyed replacement", keyed_replacement); +} + +#[test] +fn emits_never_follow_the_terminal_message_or_a_cancel() { + check("emit vs terminal and cancel", emit_vs_terminal_and_cancel); +} diff --git a/concurrency/tests/delivery_simulation.rs b/concurrency/tests/delivery_simulation.rs index ccf30dbb7..b1415e525 100644 --- a/concurrency/tests/delivery_simulation.rs +++ b/concurrency/tests/delivery_simulation.rs @@ -146,7 +146,7 @@ impl<'a> World<'a> { if let Some(reply) = pending.take() { assert!(matches!( reply.wait(self.rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Normal)) )); } } diff --git a/concurrency/tests/execution_groups.rs b/concurrency/tests/execution_groups.rs index 294671e55..3b79eff73 100644 --- a/concurrency/tests/execution_groups.rs +++ b/concurrency/tests/execution_groups.rs @@ -26,15 +26,29 @@ impl Message for Ping { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) -> String { - std::thread::current() - .name() - .unwrap_or_default() - .to_string() + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, String>, + _: &Context, + ) { + reply.reply( + std::thread::current() + .name() + .unwrap_or_default() + .to_string(), + ); } } impl Handler for Worker { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } // Not simulated: asserts OS worker thread names. @@ -107,7 +121,10 @@ struct Starting { stopped: Arc, } impl Actor for Starting { - async fn started(&mut self, _: &Context) { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); self.entered.take().unwrap().send(()).unwrap(); let _ = self.release.take().unwrap().await; } @@ -138,10 +155,16 @@ async fn expired_shutdown_retains_closed_runtime_until_startup_cleanup_finishes( assert_eq!(report.unfinished_groups, ["slow-start"]); assert!(group.is_closed()); assert!(Worker.start_on(&group).is_err()); + // The system kills what remains at grace (E3); let the kill land before + // started() returns, so the outcome does not race it. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); assert!(within!(actor.ready()).await.is_err()); - assert_eq!(stopped.load(Ordering::SeqCst), 1); + // The kill at grace (5 ms) arrived during started(): it ends Killed and + // skips the stopped() that had not begun (E1, decision 2). + assert_eq!(actor.wait_exit().await, ExitReason::Killed); + assert_eq!(stopped.load(Ordering::SeqCst), 0); } struct Probe(Arc); @@ -316,7 +339,7 @@ async fn ambient_actor_started_inside_group_is_unmanaged_and_aborted_at_teardown .unwrap() .starts_with("spawned-ambient-child")); assert!(system.shutdown(LIMIT).await.complete); - assert_eq!(within!(ambient.wait_exit()).await, ExitReason::Kill); + assert_eq!(within!(ambient.wait_exit()).await, ExitReason::Killed); assert_eq!(within!(managed.wait_exit()).await, ExitReason::Shutdown); } @@ -341,9 +364,15 @@ impl Actor for FailedStartup { } } impl Handler for FailedStartup { - async fn handle(&mut self, msg: Queued, _: &Context) { + async fn handle( + &mut self, + msg: Queued, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.handled.fetch_add(1, Ordering::SeqCst); drop(msg.0); + reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -367,7 +396,12 @@ async fn failed_startup_discards_queued_payloads_and_reply_channels_without_disp .await .unwrap() .unwrap(); - let reply = actor.request_raw(Queued(Probe(dropped.clone()))).unwrap(); + let reply = actor + .send_request( + Queued(Probe(dropped.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); actor.send(Queued(Probe(dropped.clone()))).unwrap(); assert_eq!(handled.load(Ordering::SeqCst), 0); release.send(()).unwrap(); @@ -458,6 +492,11 @@ async fn stuck_standalone_cleanup_does_not_leave_earlier_actor_running_forever() release: Option>, } impl Actor for Stuck { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { let _ = self.entered.take().unwrap().send(()); let _ = self.release.take().unwrap().await; @@ -476,11 +515,13 @@ async fn stuck_standalone_cleanup_does_not_leave_earlier_actor_running_forever() .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, first.wait_exit()) - .await - .unwrap(), - ExitReason::Shutdown + // Signalled after the grace expired, it also gets the kill at grace (E3). + let reason = spawned_rt::tasks::timeout(LIMIT, first.wait_exit()) + .await + .unwrap(); + assert!( + matches!(reason, ExitReason::Shutdown | ExitReason::Killed), + "{reason:?}" ); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); @@ -526,11 +567,27 @@ async fn readiness_reports_actual_panic_after_startup_stop_instead_of_early_norm assert!(system.shutdown(LIMIT).await.complete); } +/// Held in a handler until released. A kill and the parent's exit both wait +/// for the current handler (E1, E3), so the actor stays unfinished at any +/// deadline. (A hold in a trapping actor's `stopped()` is skipped when the +/// zero-grace kill lands before the parent's exit is dispatched: E1, +/// decision 2.) struct SnapshotHeld { release: Option>, } -impl Actor for SnapshotHeld { - async fn stopped(&mut self, _: &Context) { +impl Actor for SnapshotHeld {} +struct Hold(tokio::sync::oneshot::Sender<()>); +impl Message for Hold { + type Result = (); +} +impl Handler for SnapshotHeld { + async fn handle( + &mut self, + msg: Hold, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.0.send(()); let _ = self.release.take().unwrap().await; } } @@ -545,6 +602,9 @@ async fn concurrent_zero_deadline_reports_include_scopes_during_shutdown_initial .start_on(&group) .unwrap(); within!(actor.ready()).await.unwrap(); + let (entered, entered_rx) = tokio::sync::oneshot::channel(); + actor.send(Hold(entered)).unwrap(); + entered_rx.await.unwrap(); let mut reporters = Vec::new(); for _ in 0..64 { let owner = system.clone(); @@ -568,8 +628,13 @@ impl Message for CheckAdmission { struct HelperAware; impl Actor for HelperAware {} impl Handler for HelperAware { - async fn handle(&mut self, _: CheckAdmission, ctx: &Context) -> bool { - ctx.spawn_helper(async {}).is_err() + async fn handle( + &mut self, + _: CheckAdmission, + reply: spawned_concurrency::PendingReply<'_, bool>, + ctx: &Context, + ) { + reply.reply(ctx.spawn_helper(async {}).is_err()); } } struct Dependent { @@ -577,6 +642,11 @@ struct Dependent { observed: Arc, } impl Actor for Dependent { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on the system's shutdown only for a + // trapping actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { assert!(self.earlier.request(CheckAdmission).await.unwrap()); self.observed.fetch_add(1, Ordering::SeqCst); diff --git a/concurrency/tests/execution_profiles.rs b/concurrency/tests/execution_profiles.rs index 819f9201b..b91c0cd7f 100644 --- a/concurrency/tests/execution_profiles.rs +++ b/concurrency/tests/execution_profiles.rs @@ -25,12 +25,20 @@ impl Message for Audit { type Result = (String, Option); } impl Handler for Probe { - async fn handle(&mut self, _: Audit, _: &Context) -> ::Result { - spawned_rt::tasks::sleep(Duration::from_millis(1)).await; - ( - std::thread::current().name().unwrap_or_default().into(), - spawned_rt::tasks::affinity::current_cpu().ok(), - ) + async fn handle( + &mut self, + _: Audit, + reply: spawned_concurrency::PendingReply<'_, ::Result>, + _: &Context, + ) { + let __reply_value: ::Result = { + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + ( + std::thread::current().name().unwrap_or_default().into(), + spawned_rt::tasks::affinity::current_cpu().ok(), + ) + }; + reply.reply(__reply_value); } } struct StartPeer; @@ -41,9 +49,12 @@ impl Handler for Probe { async fn handle( &mut self, _: StartPeer, + reply: spawned_concurrency::PendingReply<'_, ::Result>, ctx: &Context, - ) -> ::Result { - ctx.execution_group().unwrap().start(Probe) + ) { + let __reply_value: ::Result = + { ctx.execution_group().unwrap().start(Probe) }; + reply.reply(__reply_value); } } // Not simulated: native execution groups need OS threads. @@ -86,11 +97,17 @@ impl Message for Block { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Block, _: &Context) { + async fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { tokio::task::block_in_place(|| { let _ = msg.entered.send(()); msg.release.recv_timeout(LIMIT).unwrap(); }); + reply.reply(()); } } struct Crash; @@ -98,8 +115,15 @@ impl Message for Crash { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("profile recovery injection"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("profile recovery injection"); + } } } // Not simulated: native execution groups need OS threads. @@ -214,17 +238,17 @@ async fn each_strategy_recovers_on_a_full_native_pool_without_overlapping_slots( let system = ActorSystem::new().unwrap(); let group = system.thread_group("supervised-roots", 3).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 20, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ), + ) .unwrap(); let calls = Arc::new(AtomicUsize::new(0)); let mut children = Vec::new(); @@ -267,6 +291,7 @@ async fn each_strategy_recovers_on_a_full_native_pool_without_overlapping_slots( SupervisorStrategy::OneForOne => 4, SupervisorStrategy::OneForAll => 6, SupervisorStrategy::RestForOne => 5, + other => unreachable!("strategy {other:?} not under test"), } ); assert!(supervisor.shutdown(LIMIT).await.complete); @@ -347,8 +372,14 @@ mod tls_join { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, msg: Install, _: &Context) { + async fn handle( + &mut self, + msg: Install, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { PROBE.with(|p| *p.borrow_mut() = Some(msg.0)); + reply.reply(()); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] diff --git a/concurrency/tests/exit_signal_matrix.rs b/concurrency/tests/exit_signal_matrix.rs index 559b96c49..5dedc8f0d 100644 --- a/concurrency/tests/exit_signal_matrix.rs +++ b/concurrency/tests/exit_signal_matrix.rs @@ -28,7 +28,9 @@ macro_rules! actors { struct Idle; impl Actor for Idle {} impl Handler for Idle { - $($asynchronous)? fn handle(&mut self, _: Ping, _: &Context) {} + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } } struct Observer(mpsc::Sender); impl Actor for Observer { @@ -37,11 +39,15 @@ macro_rules! actors { } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, _: Ping, _: &Context) {} + $($asynchronous)? fn handle(&mut self, _: Ping, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } } impl Handler for Observer { - $($asynchronous)? fn handle(&mut self, down: Down, _: &Context) { + $($asynchronous)? fn handle(&mut self, down: Down, reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context) { + self.0.send(Notice::Down(down.reason)).unwrap(); +reply.reply(()); } } }; diff --git a/concurrency/tests/exit_signals_sim.rs b/concurrency/tests/exit_signals_sim.rs new file mode 100644 index 000000000..25b376ccb --- /dev/null +++ b/concurrency/tests/exit_signals_sim.rs @@ -0,0 +1,616 @@ +//! Simulator orderings for OTP exit signals (docs/design/otp-exit-signals.md), +//! explored with PCT and bounded DFS over real actors: a kill racing +//! `stopped()`, a late signal racing the death point D, a link or monitor +//! racing D (real notice or `NoProc`), the unlink/demonitor fences against +//! delivery, a supervisor's shutdown (the parent's exit) against a trapping +//! child's queued messages, and S1's deferral of a ready child's exit racing +//! the tree's commit. Run with: +//! +//! ```sh +//! RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" cargo test -p spawned-concurrency --test exit_signals_sim +//! ``` +#![cfg(all(spawned_sim, feature = "async"))] + +use spawned_concurrency::{ + message::Message, + reply::PendingReply, + supervision::{ChildSpec, SupervisorConfig, SupervisorStrategy}, + system::ActorSystem, + tasks::{Actor, ActorRef, ActorStart, Context, Handler}, + Down, Exit, ExitReason, MonitorRef, +}; +use spawned_rt::tasks::{ + sim::explore::{explore, Strategy}, + sleep, spawn, +}; +use std::{ + collections::BTreeMap, + future::Future, + sync::{Arc, Mutex}, + time::Duration, +}; + +type Log = Arc>>; + +fn push(log: &Log, entry: impl Into) { + log.lock().unwrap().push(entry.into()); +} + +fn entries(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +fn position(log: &[String], entry: &str) -> Option { + log.iter().position(|e| e == entry) +} + +// Outcome counts per body, printed by the driver: every branch the +// assertions distinguish must be reached. Keyed by test: the simulator runs +// each body on a fresh thread named like the test thread that explores it. +static OUTCOMES: std::sync::Mutex>> = + std::sync::Mutex::new(BTreeMap::new()); + +fn test_name() -> String { + std::thread::current().name().unwrap_or("").to_owned() +} + +fn outcomes() -> std::sync::MutexGuard<'static, BTreeMap>> { + OUTCOMES.lock().unwrap_or_else(|p| p.into_inner()) +} + +fn outcome(name: impl Into) { + *outcomes() + .entry(test_name()) + .or_default() + .entry(name.into()) + .or_default() += 1; +} + +fn ms(n: u64) -> Duration { + Duration::from_millis(n) +} + +/// A logging actor: `stopped()` takes `slow_stop` virtual milliseconds. +struct Node { + log: Log, + trap: bool, + slow_stop: u64, +} + +impl Node { + fn new(log: &Log, trap: bool) -> Self { + Self { + log: log.clone(), + trap, + slow_stop: 0, + } + } +} + +impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, ctx: &Context) { + push(&self.log, format!("stopped:begin:{}", ctx.exit_pending())); + for _ in 0..self.slow_stop { + sleep(ms(1)).await; + } + push(&self.log, "stopped:end"); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + push(&self.log, format!("exit:{:?}", exit.reason)); + } +} + +impl Handler for Node { + async fn handle(&mut self, down: Down, reply: PendingReply<'_, ()>, _: &Context) { + push(&self.log, format!("down:{:?}", down.reason)); + reply.reply(()); + } +} + +type Action = Box) + Send>; +struct Do(Action); +impl Message for Do { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, f: Do, reply: PendingReply<'_, ()>, ctx: &Context) { + (f.0)(ctx); + reply.reply(()); + } +} + +/// Logged when handled; records whether an exit is pending. +struct Probe(u32); +impl Message for Probe { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, p: Probe, reply: PendingReply<'_, ()>, ctx: &Context) { + push(&self.log, format!("probe:{}:{}", p.0, ctx.exit_pending())); + reply.reply(()); + } +} + +async fn on(actor: &ActorRef, f: impl FnOnce(&Context) + Send + 'static) { + actor.request(Do(Box::new(f))).await.unwrap(); +} + +/// A round trip, so everything queued before it was handled. +async fn settle(actor: &ActorRef) { + on(actor, |_| {}).await; +} + +async fn stop(actor: &ActorRef) { + let handle = actor.child_handle(); + handle.stop(); + handle.wait_exit_async().await; +} + +/// A `Down` is enqueued by the monitor's waiter after the target's exit is +/// published (unlike a link notice, not before `wait_exit` returns): wait +/// for it, bounded, then settle once more so a second one would show. +async fn downs(observer: &ActorRef, log: &Log) -> Vec { + for _ in 0..100 { + settle(observer).await; + if !notices(&entries(log), "down:").is_empty() { + break; + } + sleep(ms(1)).await; + } + settle(observer).await; + notices(&entries(log), "down:") +} + +fn notices(log: &[String], kind: &str) -> Vec { + log.iter() + .filter(|e| e.starts_with(kind)) + .cloned() + .collect() +} + +// --------------------------------------------------------------------------- +// Bodies +// --------------------------------------------------------------------------- + +/// E1: a kill racing an administrative stop's slow `stopped()`. Before +/// `stopped()` begins it is skipped and the actor ends `Killed`; during it, +/// it finishes and the actor ends `Killed`; after D it is ignored. A monitor +/// sees the same final reason. +async fn kill_vs_stopped(seed: u64) { + let (log, watched) = (Log::default(), Log::default()); + let mut node = Node::new(&log, false); + node.slow_stop = 2; + let target = node.start(); + let watcher = Node::new(&watched, false).start(); + let handle = target.child_handle(); + let h = handle.clone(); + on(&watcher, move |ctx| { + ctx.monitor(&h); + }) + .await; + let (stopper, killer) = (handle.clone(), handle.clone()); + let stop_at = seed % 3; + let kill_at = (seed / 3) % 5; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Shutdown); + }); + let l = log.clone(); + let killing = spawn(async move { + sleep(ms(kill_at)).await; + push(&l, "kill"); + killer.kill(); + }); + let reason = handle.wait_exit_async().await; + stopping.await.unwrap(); + killing.await.unwrap(); + let log = entries(&log); + let kill = position(&log, "kill").unwrap(); + let begin = log.iter().position(|e| e.starts_with("stopped:begin")); + let end = position(&log, "stopped:end"); + assert_eq!( + begin.is_some(), + end.is_some(), + "a begun stopped() finishes: {log:?}" + ); + assert!( + begin.is_none_or(|b| b < kill), + "stopped() began after the kill: {log:?}" + ); + outcome(match (&reason, begin) { + (ExitReason::Killed, None) => "kill: before stopped()", + (ExitReason::Killed, Some(_)) => "kill: during stopped()", + _ => "kill: after D", + }); + match reason { + ExitReason::Killed => {} + ExitReason::Shutdown => { + assert!( + end.is_some_and(|e| e < kill), + "the kill came before D but did not win: {log:?}" + ); + } + other => panic!("unexpected reason {other:?}: {log:?}"), + } + assert_eq!( + downs(&watcher, &watched).await, + vec![format!("down:{reason:?}")] + ); + stop(&watcher).await; +} + +/// Decision 8: an untrapped signal racing D after a `Normal` stop replaces +/// the reason until D (the running `stopped()` still finishes); after D it +/// is ignored. A trapping link and a monitor see the same final reason. +async fn late_signal_vs_death(seed: u64) { + let (log, observed) = (Log::default(), Log::default()); + let mut node = Node::new(&log, false); + node.slow_stop = 2; + let target = node.start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let h = handle.clone(); + on(&observer, move |ctx| { + ctx.link(&h); + ctx.monitor(&h); + }) + .await; + handle.stop(); + let (signaller, l) = (handle.clone(), log.clone()); + let at = seed % 5; + let signalling = spawn(async move { + sleep(ms(at)).await; + push(&l, "signal"); + signaller.exit(ExitReason::Shutdown); + }); + let reason = handle.wait_exit_async().await; + signalling.await.unwrap(); + let log = entries(&log); + let signal = position(&log, "signal").unwrap(); + let end = position(&log, "stopped:end").expect("a self-exit runs stopped()"); + outcome(format!("late signal: {reason:?}")); + match reason { + ExitReason::Shutdown => {} + ExitReason::Normal => assert!(end < signal, "a signal before D was lost: {log:?}"), + other => panic!("unexpected reason {other:?}: {log:?}"), + } + let downs = downs(&observer, &observed).await; + assert_eq!( + notices(&entries(&observed), "exit:"), + vec![format!("exit:{reason:?}")] + ); + assert_eq!(downs, vec![format!("down:{reason:?}")]); + stop(&observer).await; +} + +/// E4/E5: `link` + `monitor` (even seeds) or `try_link` (odd seeds) racing +/// the target's D. Each registration gets exactly one notice, the real +/// reason before D or `NoProc` after it; registrations made in one turn +/// agree; `try_link` after D gets `Err(NoProc)` and no notice. +async fn link_vs_death(seed: u64) { + let observed = Log::default(); + let target = Node::new(&Log::default(), false).start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let stopper = handle.clone(); + let stop_at = seed % 3; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Shutdown); + }); + sleep(ms((seed / 3) % 3)).await; + let use_try = seed % 2 == 1; + let result = Arc::new(Mutex::new(None)); + let (h, r) = (handle.clone(), result.clone()); + on(&observer, move |ctx| { + if use_try { + *r.lock().unwrap() = Some(ctx.try_link(&h).is_ok()); + } else { + ctx.link(&h); + ctx.monitor(&h); + } + }) + .await; + stopping.await.unwrap(); + assert_eq!(handle.wait_exit_async().await, ExitReason::Shutdown); + let downs = if use_try { + settle(&observer).await; + Vec::new() + } else { + downs(&observer, &observed).await + }; + let observed = entries(&observed); + let exits = notices(&observed, "exit:"); + outcome(format!( + "link: try={use_try} {:?}", + exits.first().map(String::as_str) + )); + if use_try { + let linked = result.lock().unwrap().unwrap(); + let expected = if linked { + vec!["exit:Shutdown".to_string()] + } else { + vec![] + }; + assert_eq!(exits, expected, "try_link linked={linked}"); + } else { + assert_eq!(exits.len(), 1, "{observed:?}"); + assert!(["exit:Shutdown", "exit:NoProc"].contains(&exits[0].as_str())); + let down = exits[0].replace("exit:", "down:"); + assert_eq!(downs, vec![down], "link and monitor in one turn disagree"); + } + stop(&observer).await; +} + +/// E6: `unlink` + `demonitor` racing the target's death and delivery. A +/// marker sent to itself right after the fence turn is behind any notice +/// enqueued before it; no `Exit` or `Down` of that link or monitor is handled +/// after the marker, and at most one of each before it. +async fn unlink_demonitor_fence(seed: u64) { + struct Marker; + impl Message for Marker { + type Result = (); + } + impl Handler for Node { + async fn handle(&mut self, _: Marker, reply: PendingReply<'_, ()>, _: &Context) { + push(&self.log, "marker"); + reply.reply(()); + } + } + let observed = Log::default(); + let target = Node::new(&Log::default(), false).start(); + let observer = Node::new(&observed, true).start(); + let handle = target.child_handle(); + let monitor: Arc>> = Arc::default(); + let (h, m) = (handle.clone(), monitor.clone()); + on(&observer, move |ctx| { + ctx.link(&h); + *m.lock().unwrap() = Some(ctx.monitor(&h)); + }) + .await; + let stopper = handle.clone(); + let stop_at = seed % 3; + let stopping = spawn(async move { + sleep(ms(stop_at)).await; + stopper.stop_with_reason(ExitReason::Panic("gone".into())); + }); + sleep(ms((seed / 3) % 3)).await; + let (h, m) = (handle.clone(), monitor.clone()); + on(&observer, move |ctx| { + ctx.unlink(&h); + ctx.demonitor(m.lock().unwrap().unwrap()); + ctx.send(Marker).unwrap(); + }) + .await; + stopping.await.unwrap(); + handle.wait_exit_async().await; + settle(&observer).await; + let observed = entries(&observed); + let marker = position(&observed, "marker").unwrap(); + assert!( + observed[marker + 1..] + .iter() + .all(|e| !e.starts_with("exit:") && !e.starts_with("down:")), + "a notice after unlink/demonitor returned: {observed:?}" + ); + let (exits, downs) = (notices(&observed, "exit:"), notices(&observed, "down:")); + assert!(exits.len() <= 1 && downs.len() <= 1, "{observed:?}"); + outcome(format!("fence: exit={} down={}", exits.len(), downs.len())); + stop(&observer).await; +} + +/// E3: a supervisor's shutdown (the parent's exit signal) racing a trapping +/// child's queued messages. Messages queued before the parent exit are +/// handled first and in order, `exit_pending` turns true once and stays +/// true, `exit_received` never sees the parent exit, `stopped()` runs and +/// sees it pending, and the child ends `Shutdown`. +async fn supervisor_shutdown_vs_queue(seed: u64) { + let log = Log::default(); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sim-e3", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let l = log.clone(); + let child = tree + .add( + ChildSpec::new("trapping", &group, move |_| { + let log = l.clone(); + async move { Ok::<_, String>(Node::new(&log, true)) } + }) + .shutdown_timeout(Duration::from_secs(5)), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let actor = child.current().unwrap(); + let handle = actor.child_handle(); + let gap = seed % 2; + let sending = spawn(async move { + for i in 0..4 { + let _ = actor.send(Probe(i)); + sleep(ms(gap)).await; + } + }); + sleep(ms((seed / 2) % 4)).await; + assert!(supervisor.shutdown(Duration::from_secs(10)).await.complete); + sending.await.unwrap(); + assert_eq!(handle.wait_exit_async().await, ExitReason::Shutdown); + let log = entries(&log); + assert!( + notices(&log, "exit:").is_empty(), + "exit_received saw the parent exit: {log:?}" + ); + assert_eq!( + log.last().map(String::as_str), + Some("stopped:end"), + "{log:?}" + ); + assert!(log.contains(&"stopped:begin:true".to_string()), "{log:?}"); + let probes: Vec<(u32, bool)> = log + .iter() + .filter_map(|e| e.strip_prefix("probe:")) + .map(|p| { + let (i, pending) = p.split_once(':').unwrap(); + (i.parse().unwrap(), pending == "true") + }) + .collect(); + assert!( + probes.windows(2).all(|w| w[0].0 < w[1].0), + "out of order: {log:?}" + ); + assert!( + probes.windows(2).all(|w| w[0].1 <= w[1].1), + "exit_pending went back to false: {log:?}" + ); + outcome(format!( + "e3: handled={} pending={}", + probes.len(), + probes.iter().filter(|p| p.1).count() + )); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); +} + +/// S1: a ready child killed while the tree's initial start is still running +/// (a slow sibling). Its exit is deferred until the tree commits ready, then +/// handled under OneForOne: `start()` succeeds, the child restarts once +/// and only after the sibling's `started()`, and the sibling is untouched. +async fn deferred_exit_vs_commit(seed: u64) { + let log = Log::default(); + let system = ActorSystem::new().unwrap(); + let group = system.async_group("sim-s1", 2).unwrap(); + let config = SupervisorConfig::default().strategy(SupervisorStrategy::OneForOne); + let mut tree = system.supervisor(config).unwrap(); + let (l, starts) = (log.clone(), Arc::new(Mutex::new(0u32))); + let first = tree + .add(ChildSpec::new("first", &group, move |_| { + let mut n = starts.lock().unwrap(); + *n += 1; + push(&l, format!("first:start:{n}")); + let log = Log::default(); + async move { Ok::<_, String>(Node::new(&log, false)) } + })) + .unwrap(); + let (l, delay) = (log.clone(), 1 + (seed / 4) % 4); + let second = tree + .add(ChildSpec::new("second", &group, move |_| { + let l = l.clone(); + async move { + sleep(ms(delay)).await; + push(&l, "second:ready"); + Ok::<_, String>(Node::new(&Log::default(), false)) + } + })) + .unwrap(); + let kill_at = seed % 4; + let (f, l) = (first.clone(), log.clone()); + let killing = spawn(async move { + let actor = f.wait_ready().await.unwrap(); + sleep(ms(kill_at)).await; + push(&l, "kill"); + actor.child_handle().kill(); + }); + let supervisor = tree + .start() + .await + .expect("a ready child's exit failed start()"); + killing.await.unwrap(); + first.wait_for_generation(1).await.unwrap(); + let log = entries(&log); + let restart = position(&log, "first:start:2").unwrap(); + let ready = position(&log, "second:ready").unwrap(); + let kill = position(&log, "kill").unwrap(); + outcome(if kill < ready { + "s1: exit during startup (deferred)" + } else { + "s1: exit after the commit" + }); + assert!( + ready < restart, + "restarted before the tree committed: {log:?}" + ); + assert!(position(&log, "first:start:3").is_none(), "{log:?}"); + assert_eq!(second.snapshot().generation, 1); + assert!(supervisor.shutdown(Duration::from_secs(10)).await.complete); + assert!(system.shutdown(Duration::from_secs(10)).await.complete); +} + +// --------------------------------------------------------------------------- +// Driver +// --------------------------------------------------------------------------- + +/// PCT draws a new seed per run; DFS keeps one input seed per search, so it +/// is repeated over the first `DFS_SEEDS` seeds. +const PCT: (&str, usize) = ("pct:d=3", 1000); +const DFS: (&str, usize) = ("dfs:bound=3,depth=300", 300); +const DFS_SEEDS: u64 = 12; + +fn check(name: &str, body: F) +where + F: Fn(u64) -> Fut + Sync, + Fut: Future, +{ + outcomes().remove(&test_name()); + let report = explore(&Strategy::parse(PCT.0).unwrap(), 0, PCT.1, &body); + assert!( + report.found.is_none(), + "{name} under {}: {:?}", + PCT.0, + report.found + ); + println!( + "{name} {}: {} runs, {} schedules", + PCT.0, report.runs, report.schedules + ); + let (mut runs, mut exhausted) = (0, 0); + for seed in 0..DFS_SEEDS { + let report = explore(&Strategy::parse(DFS.0).unwrap(), seed, DFS.1, &body); + assert!( + report.found.is_none(), + "{name} under {} seed {seed}: {:?}", + DFS.0, + report.found + ); + runs += report.runs; + exhausted += usize::from(report.complete()); + } + println!( + "{name} {}: {runs} runs over {DFS_SEEDS} seeds, {exhausted} searches exhausted", + DFS.0 + ); + let outcomes = outcomes().remove(&test_name()).unwrap_or_default(); + assert!(!outcomes.is_empty(), "{name}: no outcome recorded"); + println!("{name} outcomes: {outcomes:?}"); +} + +#[test] +fn kill_races_stopped() { + check("kill vs stopped", kill_vs_stopped); +} + +#[test] +fn late_signal_races_the_death_point() { + check("late signal vs D", late_signal_vs_death); +} + +#[test] +fn link_and_monitor_race_the_death_point() { + check("link vs D", link_vs_death); +} + +#[test] +fn unlink_and_demonitor_fence_delivery() { + check("unlink/demonitor fence", unlink_demonitor_fence); +} + +#[test] +fn supervisor_shutdown_races_a_trapping_childs_queue() { + check("supervisor shutdown vs queue", supervisor_shutdown_vs_queue); +} + +#[test] +fn ready_child_exit_races_the_tree_commit() { + check("deferred exit vs commit", deferred_exit_vs_commit); +} diff --git a/concurrency/tests/failpoints.rs b/concurrency/tests/failpoints.rs index 4e5a759b2..5de119c42 100644 --- a/concurrency/tests/failpoints.rs +++ b/concurrency/tests/failpoints.rs @@ -30,17 +30,15 @@ use std::{ const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ) } /// Wait until a site has been reached `n` times. @@ -71,6 +69,8 @@ impl Started { } struct Worker { started: Started, + /// Trap exits, so stopped() runs on a parent's shutdown (v0.8, E3). + trap: bool, name: Option<&'static str>, registration: Arc>>, } @@ -78,6 +78,7 @@ impl Worker { fn new(started: &Started) -> Self { Self { started: started.clone(), + trap: false, name: None, registration: Default::default(), } @@ -85,6 +86,9 @@ impl Worker { } impl Actor for Worker { async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } self.started .0 .lock() @@ -96,12 +100,24 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected child failure"); } } impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } // ---------------------------------------------------------------- child.factory @@ -109,7 +125,6 @@ impl Handler for Worker { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn factory_errors_restart_within_budget_and_count_generations() { let _scenario = FailScenario::setup(); - failpoints::configure("child.factory", FailPoint::new(Action::Error).times(3)).unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-factory", 1).unwrap(); let started = Started::default(); @@ -126,13 +141,17 @@ async fn factory_errors_restart_within_budget_and_count_generations() { })) .unwrap(); let tree = within!(tree.start()).await.unwrap(); - let current = child.wait_ready().await.unwrap(); + // v0.8 (S1): a first start is never retried, so the failures are + // injected into the replacements after a crash. + failpoints::configure("child.factory", FailPoint::new(Action::Error).times(3)).unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + let current = child.wait_for_generation(1).await.unwrap(); // Three injected failures, each its own generation, then one success. assert_eq!(failpoints::stats("child.factory").fired, 3); - assert_eq!(child.snapshot().generation, 4); - assert_eq!(constructed.load(Ordering::SeqCst), 1); - assert_eq!(started.handles().len(), 1); - assert_eq!(started.handles()[0].id(), current.id()); + assert_eq!(child.snapshot().generation, 5); + assert_eq!(constructed.load(Ordering::SeqCst), 2); + assert_eq!(started.handles().len(), 2); + assert_eq!(started.handles()[1].id(), current.id()); assert!(tree.failure().is_none()); let report = tree.shutdown(LIMIT).await; assert!(report.complete && report.failure.is_none()); @@ -140,7 +159,7 @@ async fn factory_errors_restart_within_budget_and_count_generations() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { +async fn persistent_factory_failure_fails_startup_without_retry() { let _scenario = FailScenario::setup(); failpoints::configure("child.factory", Action::Error).unwrap(); let system = ActorSystem::new().unwrap(); @@ -155,13 +174,14 @@ async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { })) .unwrap(); match within!(tree.start()).await { - Err(SupervisorError::Failed(error)) => { - assert!(error.contains("restart budget exhausted"), "{error}") + Err(SupervisorError::FailedToStartChild { child, cause, .. }) => { + assert_eq!(child, "worker"); + assert!(cause.contains("failpoint child.factory"), "{cause}") } - other => panic!("expected budget failure, got {:?}", other.err()), + other => panic!("expected a first-start failure, got {:?}", other.err()), } - // One initial attempt plus max_restarts retries. - assert_eq!(failpoints::stats("child.factory").fired, 3); + // v0.8 (S1): one attempt; a first start is not retried. + assert_eq!(failpoints::stats("child.factory").fired, 1); assert!(started.handles().is_empty()); assert!(child.current().is_err()); assert!(system.shutdown(LIMIT).await.complete); @@ -170,7 +190,6 @@ async fn persistent_factory_failure_exhausts_budget_and_fails_startup() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn factory_panic_is_a_failed_start_not_a_controller_failure() { let _scenario = FailScenario::setup(); - failpoints::configure("child.factory", FailPoint::new(Action::Panic).times(1)).unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-factory-panic", 1).unwrap(); let started = Started::default(); @@ -183,9 +202,12 @@ async fn factory_panic_is_a_failed_start_not_a_controller_failure() { })) .unwrap(); let tree = within!(tree.start()).await.unwrap(); - child.wait_ready().await.unwrap(); + // v0.8 (S1): injected into the replacement; a first start is not retried. + failpoints::configure("child.factory", FailPoint::new(Action::Panic).times(1)).unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + child.wait_for_generation(1).await.unwrap(); let snapshot = child.snapshot(); - assert_eq!(snapshot.generation, 2); + assert_eq!(snapshot.generation, 3); assert!(snapshot .last_error .as_deref() @@ -198,11 +220,6 @@ async fn factory_panic_is_a_failed_start_not_a_controller_failure() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn seeded_probabilistic_factory_failures_match_generation_count() { let _scenario = FailScenario::setup(); - failpoints::configure( - "child.factory", - FailPoint::new(Action::Error).probability(0.5), - ) - .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-probability", 1).unwrap(); let started = Started::default(); @@ -216,6 +233,13 @@ async fn seeded_probabilistic_factory_failures_match_generation_count() { .unwrap(); let tree = within!(tree.start()).await.unwrap(); let mut actor = child.wait_ready().await.unwrap(); + // Armed after the initial start: since S1 any failed first start fails + // `start()`, so only replacements may fail (and are retried). + failpoints::configure( + "child.factory", + FailPoint::new(Action::Error).probability(0.5), + ) + .unwrap(); for _ in 0..8 { let generation = child.snapshot().generation; actor.send(Crash).unwrap(); @@ -240,11 +264,6 @@ async fn seeded_probabilistic_factory_failures_match_generation_count() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn ready_generation_discarded_before_publication_is_never_current() { let _scenario = FailScenario::setup(); - failpoints::configure( - "child.after_ready_before_publish", - FailPoint::new(Action::Error).times(1), - ) - .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-discard", 1).unwrap(); let started = Started::default(); @@ -257,17 +276,25 @@ async fn ready_generation_discarded_before_publication_is_never_current() { })) .unwrap(); let tree = within!(tree.start()).await.unwrap(); - let current = child.wait_ready().await.unwrap(); + // v0.8 (S1): a discarded first start fails start(), so discard a + // replacement. + failpoints::configure( + "child.after_ready_before_publish", + FailPoint::new(Action::Error).times(1), + ) + .unwrap(); + child.wait_ready().await.unwrap().send(Crash).unwrap(); + let current = child.wait_for_generation(1).await.unwrap(); let handles = started.handles(); - assert_eq!(handles.len(), 2); - assert_ne!(handles[0].id(), current.id()); - assert_eq!(handles[1].id(), current.id()); + assert_eq!(handles.len(), 3); + assert_ne!(handles[1].id(), current.id()); + assert_eq!(handles[2].id(), current.id()); // The discarded generation was stopped, not leaked. - let reason = tokio::time::timeout(LIMIT, handles[0].wait_exit_async()) + let reason = tokio::time::timeout(LIMIT, handles[1].wait_exit_async()) .await .unwrap(); assert_eq!(reason, ExitReason::Shutdown); - assert_eq!(child.snapshot().generation, 2); + assert_eq!(child.snapshot().generation, 3); assert!(tree.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); } @@ -528,20 +555,14 @@ async fn placement_failure_is_reported_and_fails_the_supervisor() { })) .unwrap(); match within!(tree.start()).await { - Err(SupervisorError::Failed(error)) => { - assert!(error.contains("placement failed"), "{error}") + Err(SupervisorError::FailedToStartChild { child, cause, .. }) => { + assert_eq!(child, "worker"); + assert!(cause.contains("placement failed"), "{cause}") } other => panic!("expected placement failure, got {:?}", other.err()), } - // start() returns once the supervisor has failed; the child's controller - // publishes its terminal state afterwards. Never started: `Failed` is - // sticky, so the outcome does not depend on whether the unstarted-child - // sweep or the controller finishes last. - let began = Instant::now(); - while child.snapshot().state == ChildState::Waiting { - assert!(began.elapsed() < LIMIT, "child never left Waiting"); - tokio::time::sleep(Duration::from_millis(2)).await; - } + // v0.8 (S2): start() returns after the rollback, so the never-started + // child is already terminal (#223 polled for it). assert_eq!(child.snapshot().state, ChildState::Failed); assert_eq!(child.snapshot().generation, 0); failpoints::remove("system.reserve"); @@ -653,21 +674,46 @@ struct Linker { } impl Actor for Linker {} impl Handler for Linker { - async fn handle(&mut self, msg: LinkTo, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } impl Handler for Linker { - async fn handle(&mut self, msg: Watch, ctx: &Context) { + async fn handle( + &mut self, + msg: Watch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.monitor(&msg.0); + reply.reply(()); } } impl Handler for Linker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for Linker { - async fn handle(&mut self, msg: Down, _: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.downs.send(msg.reason); + reply.reply(()); } } @@ -694,7 +740,7 @@ async fn peer_exit_during_link_install_still_propagates() { } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn target_exit_during_monitor_install_still_delivers_down() { +async fn target_exit_during_monitor_install_delivers_noproc() { let _scenario = FailScenario::setup(); failpoints::configure( "monitor.before_install", @@ -708,11 +754,13 @@ async fn target_exit_during_monitor_install_still_delivers_down() { reached("monitor.before_install", 1).await; target.send(Crash).unwrap(); within!(target.wait_exit()).await; + // The registration completes after the target's death point: exactly + // one Down, carrying NoProc (E4). let reason = tokio::time::timeout(LIMIT, received.recv()) .await .unwrap() .unwrap(); - assert!(matches!(reason, ExitReason::Panic(_))); + assert_eq!(reason, ExitReason::NoProc); watcher.context().stop(); within!(watcher.join()).await; } @@ -739,11 +787,16 @@ async fn injected_started_panic_is_an_abnormal_exit_seen_by_monitors() { other => panic!("expected panic exit, got {other:?}"), } assert!(started.handles().is_empty()); + // The monitor races the target's death point: before it, the Down + // carries the panic; after it, NoProc. let reason = tokio::time::timeout(LIMIT, received.recv()) .await .unwrap() .unwrap(); - assert!(matches!(reason, ExitReason::Panic(_))); + assert!( + matches!(reason, ExitReason::Panic(_) | ExitReason::NoProc), + "got {reason:?}" + ); watcher.context().stop(); within!(watcher.join()).await; } @@ -775,14 +828,18 @@ async fn slow_stopped_reports_at_deadline_then_completes() { .unwrap(); let system = ActorSystem::new().unwrap(); let group = system.async_group("fp-stopped", 1).unwrap(); - let actor = group.start(Worker::new(&Started::default())).unwrap(); + let mut worker = Worker::new(&Started::default()); + worker.trap = true; + let actor = group.start(worker).unwrap(); within!(actor.ready()).await.unwrap(); let report = system.shutdown(Duration::from_millis(100)).await; assert!(!report.complete); assert_eq!(report.unfinished_actors, [actor.id()]); let report = report.wait(LIMIT).await; assert!(report.complete); - assert_eq!(actor.exit_reason(), Some(ExitReason::Shutdown)); + // The kill at grace arrived while stopped() ran: it finishes, but the + // reason becomes Killed (E1). + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -879,6 +936,560 @@ async fn system_shutdown_coordinator_panic_is_reported_and_contained() { // ------------------------------------------------------------ actor.* (threads) +// ------------------------------------------------- actor.at_death (D fence) + +/// Holds deferred tokens for an outside replier. +#[derive(Default)] +struct Holder; +struct Hold(std::sync::mpsc::Sender>); +impl Message for Hold { + type Result = u32; +} +impl Actor for Holder {} +impl Handler for Holder { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, u32>, + _: &Context, + ) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } +} +impl Handler for Holder { + async fn handle( + &mut self, + _: Ping, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + } +} + +/// Review P1 (Codex 4): once `NoProc` is observable (a send to the dead +/// owner fails), an external `ReplyTo::reply` must not succeed: the caller +/// gets the owner's final reason. Replies were admitted until the registry +/// closed the slot, after D: a reply could land after `NoProc`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn no_reply_succeeds_after_noproc_is_observable() { + let _scenario = FailScenario::setup(); + let holder = Holder.start(); + let (tokens, token) = std::sync::mpsc::channel(); + let caller = holder + .send_request(Hold(tokens), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + failpoints::configure( + "actor.at_death", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + holder.context().stop(); + let began = Instant::now(); + while holder.send(Ping).is_ok() { + assert!(began.elapsed() < LIMIT, "the owner never died"); + tokio::time::sleep(Duration::from_millis(1)).await; + } + // `NoProc` is observable: the death point passed. + let delivery = token.reply(7); + assert_ne!( + delivery, + spawned_concurrency::Delivery::Delivered, + "a reply succeeded after NoProc was observable" + ); + assert!( + matches!( + caller.await, + Err(spawned_concurrency::ActorError::ActorStopped( + ExitReason::Normal + )) + ), + "the caller did not get the final reason" + ); + holder.join().await; +} + +// ---------------------------------------- actor.exit_before_deliver (fence) + +/// Traps exits and records them in arrival order. +struct Recorder(Arc>>); +impl Actor for Recorder { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + async fn exit_received(&mut self, exit: spawned_concurrency::Exit, _: &Context) { + self.0.lock().unwrap().push((exit.from, exit.reason)); + } +} +impl Handler for Recorder { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +struct SignalSender; +impl Actor for SignalSender {} +impl Handler for SignalSender { + async fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + ctx.link(&msg.0); + reply.reply(()); + } +} + +/// Review P1 (Codex 10): a signal a cloned context sends before its +/// sender's death reaches the target before the sender's death notice +/// (`ExitAfterMessages`, OTP's per-pair signal order). A helper passed the +/// sender-lifetime check, paused, and delivered after the death notice. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_exit_signal_is_not_overtaken_by_its_senders_death_notice() { + let _scenario = FailScenario::setup(); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = Recorder(seen.clone()).start(); + recorder.request(Ping).await.unwrap(); // trapping now + let sender = SignalSender.start(); + sender + .request(LinkTo(recorder.child_handle())) + .await + .unwrap(); + failpoints::configure( + "actor.exit_before_deliver", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + let ctx = sender.context(); + let target = recorder.child_handle(); + let helper = std::thread::spawn(move || ctx.exit(&target, ExitReason::Shutdown)); + reached("actor.exit_before_deliver", 1).await; + // The sender dies while its helper's signal is in flight. + sender.context().stop(); + let began = Instant::now(); + while seen.lock().unwrap().len() < 2 { + assert!( + began.elapsed() < LIMIT, + "exits seen: {:?}", + seen.lock().unwrap() + ); + tokio::time::sleep(Duration::from_millis(2)).await; + } + helper.join().unwrap(); + let reasons: Vec<_> = seen + .lock() + .unwrap() + .iter() + .map(|(_, r)| r.clone()) + .collect(); + assert_eq!( + reasons, + [ExitReason::Shutdown, ExitReason::Normal], + "the death notice overtook the sender's own signal" + ); + recorder.context().stop(); + recorder.join().await; +} + +// ---------------------------------- monitor.down_before_dispatch (flush) + +struct WatchDown(ChildHandle); +impl Message for WatchDown { + type Result = spawned_concurrency::MonitorRef; +} + +/// Monitors on request and counts the `Down`s its handler runs. +struct Observer(Arc); +impl Actor for Observer {} +impl Handler for Observer { + async fn handle( + &mut self, + msg: WatchDown, + reply: spawned_concurrency::PendingReply<'_, spawned_concurrency::MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); + } +} +impl Handler for Observer { + async fn handle( + &mut self, + _: Down, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} +impl Handler for Observer { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +struct Quiet; +impl Actor for Quiet {} + +/// OTP `demonitor(Ref, [flush])`: a `demonitor` on another thread returns +/// while the observer is dispatching the monitor's `Down` (paused before the +/// flush check); the `Down` never reaches `Handler`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn demonitor_flushes_a_down_in_dispatch() { + let _scenario = FailScenario::setup(); + let handled = Arc::new(AtomicUsize::new(0)); + let observer = Observer(handled.clone()).start(); + let target = Quiet.start(); + let monitor = observer + .request(WatchDown(target.child_handle())) + .await + .unwrap(); + failpoints::configure( + "monitor.down_before_dispatch", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + target.context().stop(); + target.join().await; + reached("monitor.down_before_dispatch", 1).await; + observer.context().demonitor(monitor); + // FIFO, one turn at a time: the Down's dispatch finished before this. + observer.request(Ping).await.unwrap(); + assert_eq!( + handled.load(Ordering::SeqCst), + 0, + "a Down was handled after demonitor returned" + ); + observer.context().stop(); + observer.join().await; +} + +// ------------------------------------- death-boundary discharge (final review) + +type SharedReceiver = Arc>>>; + +/// An unread reply whose destructor waits (at most 3 s) for a queued +/// caller's answer and records whether it came. +struct WaitsForCaller { + caller: SharedReceiver, + answered: Arc, +} +impl Drop for WaitsForCaller { + fn drop(&mut self) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if let Some(receiver) = self.caller.lock().unwrap().as_mut() { + if receiver.try_recv().is_some() { + self.answered.store(1, Ordering::SeqCst); + return; + } + } + std::thread::sleep(Duration::from_millis(1)); + } + } +} +/// Deferred, then answered at once with a `WaitsForCaller` nobody reads. +struct AnswerLater(WaitsForCaller); +impl Message for AnswerLater { + type Result = WaitsForCaller; +} +/// Blocks the actor until released; reports entry. +struct Gate(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); +impl Message for Gate { + type Result = (); +} +struct Answer; +impl Message for Answer { + type Result = (); +} +struct Discharger; +impl Actor for Discharger {} +impl Handler for Discharger { + async fn handle( + &mut self, + msg: AnswerLater, + reply: spawned_concurrency::PendingReply<'_, WaitsForCaller>, + _: &Context, + ) { + reply.defer().unwrap().reply(msg.0); + } +} +impl Handler for Discharger { + async fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } +} +impl Handler for Discharger { + async fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} + +/// Final review (Codex P1 1): the death point's close holds an upgraded, +/// already-answered deferred slot while its caller drops the receiver. The +/// close's reference was then the last: the unread reply's destructor ran +/// inside the close, before the queued caller was answered (and here waited +/// for that answer). Every slot the discharge meets is now retained until +/// every caller is answered. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unread_reply_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let actor = Discharger.start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let unread = actor + .send_request( + AnswerLater(WaitsForCaller { + caller: caller.clone(), + answered: answered.clone(), + }), + spawned_concurrency::Deadline::Never, + ) + .unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); // AnswerLater was handled + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure( + "reply.close_upgraded", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + reached("reply.close_upgraded", 1).await; + // The caller gives up on its unread reply while the close holds it. + drop(unread); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "an unread reply was destroyed before every caller was answered" + ); +} + +/// Final review (Codex P1 3): the discharge batch wakes parked blocking +/// receivers before any arbitrary waker. Caller A's waker waits for caller +/// B, parked in `recv()`, to return; B used to be notified only after A's +/// waker, so neither proceeded. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn parked_receivers_wake_before_any_waker() { + use std::task::{Wake, Waker}; + struct WaitForB(Arc, Arc); + impl Wake for WaitForB { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.0.load(Ordering::SeqCst) == 1 { + self.1.store(1, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let _scenario = FailScenario::setup(); + let actor = Discharger.start(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let a = actor.send_request(Answer, never).unwrap(); + let b = actor.send_request(Answer, never).unwrap(); + let (b_returned, a_saw_b) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let waker = Waker::from(Arc::new(WaitForB(b_returned.clone(), a_saw_b.clone()))); + let mut a = Box::pin(std::future::IntoFuture::into_future(a)); + assert!( + std::future::Future::poll(a.as_mut(), &mut std::task::Context::from_waker(&waker)) + .is_pending() + ); + failpoints::configure("reply.recv_parked", FailPoint::new(Action::Off)).unwrap(); + let parked = { + let b_returned = b_returned.clone(); + std::thread::spawn(move || { + let _ = b.recv(); + b_returned.store(1, Ordering::SeqCst); + }) + }; + reached("reply.recv_parked", 1).await; // B is parked in recv() + actor.context().stop(); + release.send(()).unwrap(); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + parked.join().unwrap(); + assert_eq!( + a_saw_b.load(Ordering::SeqCst), + 1, + "a parked blocking receiver was woken only after another caller's waker" + ); +} + +// ------------------------------- job.emit_due (exit holds payload drops) + +/// A message whose destructor records, when it starts, whether a queued +/// caller was already answered. +struct Probe { + caller: SharedReceiver, + answered: Arc, + started: std::sync::mpsc::Sender<()>, +} +impl Drop for Probe { + fn drop(&mut self) { + if let Some(receiver) = self.caller.lock().unwrap().as_mut() { + if receiver.try_recv().is_some() { + self.answered.store(1, Ordering::SeqCst); + } + } + let _ = self.started.send(()); + } +} +struct Probed(#[allow(dead_code)] Probe); +impl Message for Probed { + type Result = (); +} +/// Its `stopped()` waits for the test's release. +struct SlowStop(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); +struct ScheduleProbe(Probe, std::sync::mpsc::Sender<()>); +impl Message for ScheduleProbe { + type Result = (); +} +impl Actor for SlowStop { + async fn stopped(&mut self, _: &Context) { + let _ = self.1.send(()); + let _ = self.0.recv_timeout(LIMIT); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + msg: ScheduleProbe, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let ScheduleProbe(probe, scheduled) = msg; + ctx.pipe( + spawned_concurrency::tasks::JobOptions::new(), + move |job| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(100); + job.emit_at(at, Probed(probe)).await.unwrap(); + let _ = scheduled.send(()); + job.cancelled().await; + }, + |_| Answer, + ) + .unwrap(); + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } +} +impl Handler for SlowStop { + async fn handle( + &mut self, + _: Probed, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + } +} + +/// Final review (agy P1 1): a scheduled `emit_at` that comes due while the +/// owner runs `stopped()` (its jobs already cancelled) is refused, and its +/// payload used to be destroyed right there on the deadline service, before +/// the queued caller was answered. The owner's payload drops are now held +/// from the start of its exit until every caller is answered. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emit_due_during_exit_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let (stop_release, stop_gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let actor = SlowStop(stop_gate, stopping_tx).start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let (started_tx, started) = std::sync::mpsc::channel(); + let (scheduled_tx, scheduled) = std::sync::mpsc::channel(); + let probe = Probe { + caller: caller.clone(), + answered: answered.clone(), + started: started_tx, + }; + actor + .request(ScheduleProbe(probe, scheduled_tx)) + .await + .unwrap(); + scheduled.recv_timeout(LIMIT).unwrap(); // the emit is armed + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure("job.emit_due", FailPoint::new(Action::Off)).unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + stopping.recv_timeout(LIMIT).unwrap(); // in stopped(), jobs cancelled + // The emit came due during stopped(). Its payload must not be destroyed + // now: give a wrong implementation the chance to, then let stopped() + // return. + reached("job.emit_due", 1).await; + let _ = started.recv_timeout(Duration::from_millis(500)); + stop_release.send(()).unwrap(); + tokio::time::timeout(LIMIT * 2, actor.join()).await.unwrap(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "a scheduled emit's payload was destroyed before every caller was answered" + ); +} + mod native { use super::*; use spawned_concurrency::threads::{ @@ -888,15 +1499,153 @@ mod native { struct Target; impl NativeActor for Target {} impl NativeHandler for Target { - fn handle(&mut self, _: Crash, _: &NativeContext) { + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &NativeContext, + ) { panic!("injected native failure"); } } + /// Holds deferred tokens for an outside replier (threads mode). + struct NativeHolder; + impl NativeActor for NativeHolder {} + impl NativeHandler for NativeHolder { + fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, u32>, + _: &NativeContext, + ) { + msg.0.send(reply.defer().unwrap()).unwrap(); + } + } + impl NativeHandler for NativeHolder { + fn handle( + &mut self, + _: Ping, + _: spawned_concurrency::PendingReply<'_, ()>, + _: &NativeContext, + ) { + } + } + + /// Review P1 (Codex 4), threads mode: no external reply succeeds once + /// `NoProc` is observable. + #[test] + fn no_reply_succeeds_after_noproc_is_observable() { + let _scenario = FailScenario::setup(); + let holder = NativeHolder.start(); + let (tokens, token) = std::sync::mpsc::channel(); + let caller = holder + .send_request(Hold(tokens), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let token = token.recv_timeout(LIMIT).unwrap(); + failpoints::configure( + "actor.at_death", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + holder.context().stop(); + let began = Instant::now(); + while holder.send(Ping).is_ok() { + assert!(began.elapsed() < LIMIT, "the owner never died"); + std::thread::sleep(Duration::from_millis(1)); + } + assert_ne!( + token.reply(7), + spawned_concurrency::Delivery::Delivered, + "a reply succeeded after NoProc was observable" + ); + assert!( + matches!( + caller.recv(), + Err(spawned_concurrency::ActorError::ActorStopped( + ExitReason::Normal + )) + ), + "the caller did not get the final reason" + ); + holder.join(); + } + + /// Traps exits and records them (threads mode). + struct NativeRecorder(Arc>>); + impl NativeActor for NativeRecorder { + fn started(&mut self, ctx: &NativeContext) { + ctx.trap_exit(true); + } + fn exit_received(&mut self, exit: spawned_concurrency::Exit, _: &NativeContext) { + self.0.lock().unwrap().push(exit.reason); + } + } + struct NativeSender; + impl NativeActor for NativeSender {} + impl NativeHandler for NativeSender { + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { + ctx.link(&msg.0); + reply.reply(()); + } + } + + /// Review P1 (Codex 10), threads mode: a cloned context's signal sent + /// before its actor's death reaches the target before the death notice. + #[test] + fn an_exit_signal_is_not_overtaken_by_its_senders_death_notice() { + let _scenario = FailScenario::setup(); + let seen = Arc::new(Mutex::new(Vec::new())); + let recorder = NativeRecorder(seen.clone()).start(); + let sender = NativeSender.start(); + sender.request(LinkTo(recorder.child_handle())).unwrap(); + failpoints::configure( + "actor.exit_before_deliver", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + let ctx = sender.context(); + let target = recorder.child_handle(); + let helper = std::thread::spawn(move || ctx.exit(&target, ExitReason::Shutdown)); + let began = Instant::now(); + while failpoints::stats("actor.exit_before_deliver").hits < 1 { + assert!(began.elapsed() < LIMIT); + std::thread::sleep(Duration::from_millis(1)); + } + sender.context().stop(); + while seen.lock().unwrap().len() < 2 { + assert!( + began.elapsed() < LIMIT, + "exits seen: {:?}", + seen.lock().unwrap() + ); + std::thread::sleep(Duration::from_millis(2)); + } + helper.join().unwrap(); + assert_eq!( + *seen.lock().unwrap(), + [ExitReason::Shutdown, ExitReason::Normal], + "the death notice overtook the sender's own signal" + ); + recorder.context().stop(); + recorder.join(); + } + struct Peer; impl NativeActor for Peer {} impl NativeHandler for Peer { - fn handle(&mut self, msg: LinkTo, ctx: &NativeContext) { + fn handle( + &mut self, + msg: LinkTo, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { ctx.link(&msg.0); + reply.reply(()); } } @@ -956,4 +1705,345 @@ mod native { std::thread::sleep(Duration::from_millis(2)); } } + + struct NativeObserver(Arc); + impl NativeActor for NativeObserver {} + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + msg: WatchDown, + reply: spawned_concurrency::reply::PendingReply<'_, spawned_concurrency::MonitorRef>, + ctx: &NativeContext, + ) { + reply.reply(ctx.monitor(&msg.0)); + } + } + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + _: Down, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + impl NativeHandler for NativeObserver { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + struct NativeQuiet; + impl NativeActor for NativeQuiet {} + + /// Threads mode: a `demonitor` on another thread returns while the + /// observer dispatches the monitor's `Down`; it is never handled. + #[test] + fn demonitor_flushes_a_down_in_dispatch() { + let _scenario = FailScenario::setup(); + let handled = Arc::new(AtomicUsize::new(0)); + let observer = NativeObserver(handled.clone()).start(); + let target = NativeQuiet.start(); + let monitor = observer.request(WatchDown(target.child_handle())).unwrap(); + failpoints::configure( + "monitor.down_before_dispatch", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + target.context().stop(); + target.join(); + let began = Instant::now(); + while failpoints::stats("monitor.down_before_dispatch").hits < 1 { + assert!(began.elapsed() < LIMIT, "the Down was not dispatched"); + std::thread::sleep(Duration::from_millis(2)); + } + observer.context().demonitor(monitor); + observer.request(Ping).unwrap(); + assert_eq!( + handled.load(Ordering::SeqCst), + 0, + "a Down was handled after demonitor returned" + ); + observer.context().stop(); + observer.join(); + } + + struct NativeDischarger; + impl NativeActor for NativeDischarger {} + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + msg: AnswerLater, + reply: spawned_concurrency::reply::PendingReply<'_, WaitsForCaller>, + _: &NativeContext, + ) { + reply.defer().unwrap().reply(msg.0); + } + } + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl NativeHandler for NativeDischarger { + fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + fn wait_hits(site: &str) { + let began = Instant::now(); + while failpoints::stats(site).hits < 1 { + assert!(began.elapsed() < LIMIT, "site {site} not reached"); + std::thread::sleep(Duration::from_millis(2)); + } + } + + /// Threads mode of `an_unread_reply_is_destroyed_after_every_caller_is_answered`. + #[test] + fn an_unread_reply_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let actor = NativeDischarger.start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let unread = actor + .send_request( + AnswerLater(WaitsForCaller { + caller: caller.clone(), + answered: answered.clone(), + }), + spawned_concurrency::Deadline::Never, + ) + .unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure( + "reply.close_upgraded", + FailPoint::new(Action::Delay(Duration::from_millis(300))).times(1), + ) + .unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + wait_hits("reply.close_upgraded"); + drop(unread); + actor.join(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "an unread reply was destroyed before every caller was answered" + ); + } + + /// Threads mode of `parked_receivers_wake_before_any_waker`. + #[test] + fn parked_receivers_wake_before_any_waker() { + use std::task::{Wake, Waker}; + struct WaitForB(Arc, Arc); + impl Wake for WaitForB { + fn wake(self: Arc) { + let began = Instant::now(); + while began.elapsed() < Duration::from_secs(3) { + if self.0.load(Ordering::SeqCst) == 1 { + self.1.store(1, Ordering::SeqCst); + return; + } + std::thread::sleep(Duration::from_millis(1)); + } + } + } + let _scenario = FailScenario::setup(); + let actor = NativeDischarger.start(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + let never = spawned_concurrency::Deadline::Never; + let a = actor.send_request(Answer, never).unwrap(); + let b = actor.send_request(Answer, never).unwrap(); + let (b_returned, a_saw_b) = (Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0))); + let waker = Waker::from(Arc::new(WaitForB(b_returned.clone(), a_saw_b.clone()))); + let mut a = Box::pin(std::future::IntoFuture::into_future(a)); + assert!( + std::future::Future::poll(a.as_mut(), &mut std::task::Context::from_waker(&waker)) + .is_pending() + ); + failpoints::configure("reply.recv_parked", FailPoint::new(Action::Off)).unwrap(); + let parked = { + let b_returned = b_returned.clone(); + std::thread::spawn(move || { + let _ = b.recv(); + b_returned.store(1, Ordering::SeqCst); + }) + }; + wait_hits("reply.recv_parked"); + actor.context().stop(); + release.send(()).unwrap(); + actor.join(); + parked.join().unwrap(); + assert_eq!( + a_saw_b.load(Ordering::SeqCst), + 1, + "a parked blocking receiver was woken only after another caller's waker" + ); + } + + /// Final review (Codex P2 4): dropping an unused blocking-job permit + /// returns its native slot on the dropping thread. With the parked + /// thread held at its exit (`job.native_worker_unused`), the next + /// reservation at `MAX_NATIVE_JOBS = 1` must still succeed; the slot + /// used to be released only when that thread ended. + #[test] + fn an_unused_permit_returns_its_slot_without_its_thread() { + struct One; + impl NativeActor for One { + const MAX_NATIVE_JOBS: Option = Some(1); + } + let _scenario = FailScenario::setup(); + failpoints::configure( + "job.native_worker_unused", + FailPoint::new(Action::Delay(Duration::from_millis(500))).times(1), + ) + .unwrap(); + let owner = One.start(); + let ctx = owner.context(); + let opts = spawned_concurrency::threads::JobOptions::new; + drop(ctx.reserve_blocking_job(opts()).unwrap()); + wait_hits("job.native_worker_unused"); // the old thread is held + let again = ctx.reserve_blocking_job(opts()); + assert!( + again.is_ok(), + "the slot waited for the unused permit's thread: {again:?}" + ); + drop(again); + ctx.stop(); + owner.join(); + } + + struct NativeSlowStop(std::sync::mpsc::Receiver<()>, std::sync::mpsc::Sender<()>); + impl NativeActor for NativeSlowStop { + fn stopped(&mut self, _: &NativeContext) { + let _ = self.1.send(()); + let _ = self.0.recv_timeout(LIMIT); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + msg: ScheduleProbe, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + ctx: &NativeContext, + ) { + let ScheduleProbe(probe, scheduled) = msg; + ctx.pipe_blocking( + spawned_concurrency::threads::JobOptions::new(), + move |job| { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(100); + job.emit_at_blocking(at, Probed(probe)).unwrap(); + let _ = scheduled.send(()); + while !job.is_cancelled() { + std::thread::sleep(Duration::from_millis(2)); + } + }, + |_| Answer, + ) + .unwrap(); + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + msg: Gate, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + let _ = msg.1.send(()); + let _ = msg.0.recv_timeout(LIMIT); + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + _: Answer, + reply: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + reply.reply(()); + } + } + impl NativeHandler for NativeSlowStop { + fn handle( + &mut self, + _: Probed, + _: spawned_concurrency::reply::PendingReply<'_, ()>, + _: &NativeContext, + ) { + } + } + + /// Threads mode of `emit_due_during_exit_is_destroyed_after_every_caller_is_answered`. + #[test] + fn emit_due_during_exit_is_destroyed_after_every_caller_is_answered() { + let _scenario = FailScenario::setup(); + let (stop_release, stop_gate) = std::sync::mpsc::channel(); + let (stopping_tx, stopping) = std::sync::mpsc::channel(); + let actor = NativeSlowStop(stop_gate, stopping_tx).start(); + let caller: SharedReceiver = Arc::default(); + let answered = Arc::new(AtomicUsize::new(0)); + let (started_tx, started) = std::sync::mpsc::channel(); + let (scheduled_tx, scheduled) = std::sync::mpsc::channel(); + let probe = Probe { + caller: caller.clone(), + answered: answered.clone(), + started: started_tx, + }; + actor.request(ScheduleProbe(probe, scheduled_tx)).unwrap(); + scheduled.recv_timeout(LIMIT).unwrap(); + let (release, gate) = std::sync::mpsc::channel(); + let (entered_tx, entered) = std::sync::mpsc::channel(); + actor.send(Gate(gate, entered_tx)).unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + *caller.lock().unwrap() = Some( + actor + .send_request(Answer, spawned_concurrency::Deadline::Never) + .unwrap(), + ); + failpoints::configure("job.emit_due", FailPoint::new(Action::Off)).unwrap(); + actor.context().stop(); + release.send(()).unwrap(); + stopping.recv_timeout(LIMIT).unwrap(); + wait_hits("job.emit_due"); + let _ = started.recv_timeout(Duration::from_millis(500)); + stop_release.send(()).unwrap(); + actor.join(); + assert_eq!( + answered.load(Ordering::SeqCst), + 1, + "a scheduled emit's payload was destroyed before every caller was answered" + ); + } } diff --git a/concurrency/tests/failure_windows.rs b/concurrency/tests/failure_windows.rs index 0b5443ea9..0580af370 100644 --- a/concurrency/tests/failure_windows.rs +++ b/concurrency/tests/failure_windows.rs @@ -52,7 +52,7 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { assert_eq!(actor.wait(&rt), ExitReason::Normal); assert!(matches!( queued.wait(&rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Normal)) )); assert!(effects.lock().unwrap().is_empty()); reached.insert(REQUIRED[0]); @@ -111,7 +111,7 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { control.release(); assert!(matches!( response.wait(&rt, LIMIT), - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); assert!(matches!(actor.wait(&rt), ExitReason::Panic(_))); assert_eq!(*effects.lock().unwrap(), vec![9]); @@ -148,11 +148,13 @@ fn controlled_failures_reach_every_required_window_on_every_backend() { cleanup.track(&target); peer.link(&target.child()); peer.stop(); - control.entered(); // Normal stop is committed while cleanup is pending. + control.entered(); // Normal stop is requested while stopped() runs. assert!(target.request(&rt, Command::Panic).is_err()); assert!(matches!(target.wait(&rt), ExitReason::Panic(_))); control.release(); - assert_eq!(peer.wait(&rt), ExitReason::Normal); + // The late abnormal link notice replaces the requested Normal + // before the death point (decision 8); stopped() still finished. + assert!(matches!(peer.wait(&rt), ExitReason::Panic(_))); reached.insert(REQUIRED[5]); } { diff --git a/concurrency/tests/foreign_panics.rs b/concurrency/tests/foreign_panics.rs index f9c4e66f9..e414d1fcb 100644 --- a/concurrency/tests/foreign_panics.rs +++ b/concurrency/tests/foreign_panics.rs @@ -182,24 +182,49 @@ mod task_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.on_crash(); + reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { - 7 + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(7); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, _: &Context) -> Bomb { - self.0.gate.wait(); - Bomb(Some(REPLY_PANIC)) + async fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::PendingReply<'_, Bomb>, + _: &Context, + ) { + let __reply_value: Bomb = { + self.0.gate.wait(); + Bomb(Some(REPLY_PANIC)) + }; + reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Stash, _: &Context) { + async fn handle( + &mut self, + _: Stash, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.stash.push(Bomb(Some(REPLY_PANIC))); + reply.reply(()); } } } @@ -217,24 +242,49 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Crash, _: &Context) { + fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.on_crash(); + reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ping, _: &Context) -> u64 { - 7 + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(7); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, _: &Context) -> Bomb { - self.0.gate.wait(); - Bomb(Some(REPLY_PANIC)) + fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::PendingReply<'_, Bomb>, + _: &Context, + ) { + let __reply_value: Bomb = { + self.0.gate.wait(); + Bomb(Some(REPLY_PANIC)) + }; + reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Stash, _: &Context) { + fn handle( + &mut self, + _: Stash, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.stash.push(Bomb(Some(REPLY_PANIC))); + reply.reply(()); } } } @@ -326,7 +376,10 @@ mod env { task.abort(); let _ = env.bounded(task); } - Started::Native(a) => drop(a.request_raw(Gift).unwrap()), + Started::Native(a) => drop( + a.send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ), } gate.release(); } @@ -387,17 +440,13 @@ mod env { let Some(group) = self.group.clone() else { return true; }; - let config = SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ); let mut tree = self.system.supervisor(config).unwrap(); let stopped = stopped.clone(); let child = tree @@ -500,7 +549,14 @@ fn subscriber_child() { println!( "{place:?} panic_in_stopped={panic_in_stopped} reason={reason:?} stopped_ran={ran}" ); - if reason != handler_panic() || !ran { + // v0.8: a panic in stopped() replaces an earlier abnormal reason + // (as a crash in OTP's terminate), unless a kill was received. + let expected = if panic_in_stopped { + ExitReason::Panic("panic in stopped(): injected stopped panic".into()) + } else { + handler_panic() + }; + if reason != expected || !ran { callback_gaps.push(format!( "{place:?} stopped_panic={panic_in_stopped}: {reason:?}, stopped() ran: {ran}" )); @@ -540,7 +596,9 @@ fn panic_hook_child() { let stopped = Arc::new(AtomicBool::new(false)); let actor = env.start(Victim::new(&stopped, false)); assert_eq!(actor.ping(&env), 7); - actor.child().shutdown(); + actor + .child() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(env.wait_exit(&actor.child()), ExitReason::Shutdown); env.finish(); } @@ -598,9 +656,8 @@ fn reply_drop_child() { "{place:?}: stopped() skipped" ); } - // A held reply whose Drop panics while state is destroyed: a normal - // stop escalates to a destructor panic, an unwinding actor keeps its - // handler panic as the reason. + // A held reply whose Drop panics while state is destroyed (after the + // death point): a cleanup failure, never the reason. for unwinding in [false, true] { let stopped = Arc::new(AtomicBool::new(false)); let mut victim = Victim::new(&stopped, false); @@ -616,9 +673,14 @@ fn reply_drop_child() { let expected = if unwinding { handler_panic() } else { - ExitReason::Panic(format!("actor state destructor: {REPLY_PANIC}")) + ExitReason::Normal }; assert_eq!(reason, expected, "{place:?} unwinding={unwinding}"); + assert_eq!( + actor.child().cleanup_failure(), + Some(format!("actor state destructor: {REPLY_PANIC}")), + "{place:?} unwinding={unwinding}" + ); } // The process and the remaining actors stay healthy. let stopped = Arc::new(AtomicBool::new(false)); diff --git a/concurrency/tests/full_actor_shards.rs b/concurrency/tests/full_actor_shards.rs index 5e00450bc..b7651fdbd 100644 --- a/concurrency/tests/full_actor_shards.rs +++ b/concurrency/tests/full_actor_shards.rs @@ -36,11 +36,18 @@ impl Message for Thread { type Result = String; } impl Handler for Worker { - async fn handle(&mut self, _: Thread, _: &Context) -> String { - std::thread::current() - .name() - .unwrap_or("unnamed") - .to_owned() + async fn handle( + &mut self, + _: Thread, + reply: spawned_concurrency::PendingReply<'_, String>, + _: &Context, + ) { + reply.reply( + std::thread::current() + .name() + .unwrap_or("unnamed") + .to_owned(), + ); } } struct Hold { @@ -51,10 +58,18 @@ impl Message for Hold { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, _: &Context) -> u64 { - let _ = msg.started.send(()); - msg.release.await.unwrap(); - 42 + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + let _ = msg.started.send(()); + msg.release.await.unwrap(); + 42 + }; + reply.reply(__reply_value); } } struct Nested(std::net::SocketAddr); @@ -62,24 +77,32 @@ impl Message for Nested { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, msg: Nested, _: &Context) -> u64 { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - let child = Worker(Default::default()).start(); - let _ = child.request(Thread).await.unwrap(); - child.child_handle().stop(); - within!(child.join()).await; - let helper = tokio::spawn(async { - tokio::time::sleep(Duration::from_millis(1)).await; - 42u64 - }); - let mut socket = tokio::net::TcpStream::connect(msg.0).await.unwrap(); - socket - .write_all(&helper.await.unwrap().to_le_bytes()) - .await - .unwrap(); - let mut reply = [0; 8]; - socket.read_exact(&mut reply).await.unwrap(); - u64::from_le_bytes(reply) + async fn handle( + &mut self, + msg: Nested, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let child = Worker(Default::default()).start(); + let _ = child.request(Thread).await.unwrap(); + child.child_handle().stop(); + within!(child.join()).await; + let helper = tokio::spawn(async { + tokio::time::sleep(Duration::from_millis(1)).await; + 42u64 + }); + let mut socket = tokio::net::TcpStream::connect(msg.0).await.unwrap(); + socket + .write_all(&helper.await.unwrap().to_le_bytes()) + .await + .unwrap(); + let mut reply = [0; 8]; + socket.read_exact(&mut reply).await.unwrap(); + u64::from_le_bytes(reply) + }; + reply.reply(__reply_value); } } fn runtime() -> tokio::runtime::Runtime { @@ -131,13 +154,18 @@ fn stop_finishes_in_flight_handler_and_rejects_queued_work() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let reply = actor - .request_raw(Hold { - started, - release: gate, - }) + .send_request( + Hold { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); - let queued = actor.request_raw(Thread).unwrap(); + let queued = actor + .send_request(Thread, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.child_handle().stop(); assert!(actor.exit_reason().is_none()); release.send(()).unwrap(); @@ -185,10 +213,13 @@ fn suspended_actor_does_not_block_another_actor_on_its_shard() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let held = actors[a] - .request_raw(Hold { - started, - release: gate, - }) + .send_request( + Hold { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); assert_eq!( @@ -242,9 +273,15 @@ impl Message for Freeze { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Freeze, _: &Context) { + async fn handle( + &mut self, + msg: Freeze, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = msg.started.send(()); msg.release.recv_timeout(LIMIT).unwrap(); + reply.reply(()); } } #[test] @@ -276,17 +313,19 @@ fn blocking_handler_stalls_its_shard_while_another_shard_progresses() { let (started, rx) = tokio::sync::oneshot::channel(); let (release, gate) = std::sync::mpsc::channel(); let frozen = actors[a] - .request_raw(Freeze { - started, - release: gate, - }) + .send_request( + Freeze { + started, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, rx).await.unwrap().unwrap(); - let mut queued = actors[b].request_raw(Thread).unwrap(); - assert!(matches!( - queued.try_recv(), - Err(tokio::sync::oneshot::error::TryRecvError::Empty) - )); + let mut queued = actors[b] + .send_request(Thread, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert!(queued.try_recv().is_none()); if let Some(other) = (0..actors.len()).find(|&id| names[id] != names[a]) { assert_eq!( actors[other] @@ -318,9 +357,17 @@ impl Message for Tick { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _: Tick, _: &Context) -> u64 { - tokio::time::sleep(Duration::from_millis(1)).await; - 42 + async fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + tokio::time::sleep(Duration::from_millis(1)).await; + 42 + }; + reply.reply(__reply_value); } } struct SpawnBlockingChild; @@ -331,9 +378,13 @@ impl Handler for Worker { async fn handle( &mut self, _: SpawnBlockingChild, + reply: spawned_concurrency::PendingReply<'_, spawned_concurrency::tasks::ActorRef>, _: &Context, - ) -> spawned_concurrency::tasks::ActorRef { - Worker(Default::default()).start_with_backend(spawned_concurrency::tasks::Backend::Blocking) + ) { + reply.reply( + Worker(Default::default()) + .start_with_backend(spawned_concurrency::tasks::Backend::Blocking), + ); } } struct Helper(tokio::sync::oneshot::Receiver<()>); @@ -341,12 +392,17 @@ impl Message for Helper { type Result = tokio::task::JoinHandle; } impl Handler for Worker { - async fn handle(&mut self, msg: Helper, _: &Context) -> tokio::task::JoinHandle { - tokio::spawn(async move { + async fn handle( + &mut self, + msg: Helper, + reply: spawned_concurrency::PendingReply<'_, tokio::task::JoinHandle>, + _: &Context, + ) { + reply.reply(tokio::spawn(async move { msg.0.await.unwrap(); tokio::time::sleep(Duration::from_millis(1)).await; 42 - }) + })); } } #[test] @@ -385,13 +441,25 @@ impl Message for InstallMonitor { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: InstallMonitor, _: &Context) { + async fn handle( + &mut self, + msg: InstallMonitor, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { msg.observer.monitor(&msg.target); + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, _: spawned_concurrency::Down, _: &Context) { + async fn handle( + &mut self, + _: spawned_concurrency::Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.lock().unwrap().push("down"); + reply.reply(()); } } struct Events; @@ -399,8 +467,13 @@ impl Message for Events { type Result = Vec<&'static str>; } impl Handler for Worker { - async fn handle(&mut self, _: Events, _: &Context) -> Vec<&'static str> { - self.0.lock().unwrap().clone() + async fn handle( + &mut self, + _: Events, + reply: spawned_concurrency::PendingReply<'_, Vec<&'static str>>, + _: &Context, + ) { + reply.reply(self.0.lock().unwrap().clone()); } } #[test] diff --git a/concurrency/tests/future_layout.rs b/concurrency/tests/future_layout.rs index 12fd0432a..f9b7ac093 100644 --- a/concurrency/tests/future_layout.rs +++ b/concurrency/tests/future_layout.rs @@ -23,11 +23,23 @@ trait ProbeProtocol: Send + Sync { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: probe_protocol::Notify, _: &Context) {} + async fn handle( + &mut self, + _: probe_protocol::Notify, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for Worker { - async fn handle(&mut self, query: probe_protocol::Query, _: &Context) -> u64 { - query.value + async fn handle( + &mut self, + query: probe_protocol::Query, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(query.value); } } const BASELINE_WORDS: usize = 1; diff --git a/concurrency/tests/job_core_loom.rs b/concurrency/tests/job_core_loom.rs new file mode 100644 index 000000000..2fbb471d8 --- /dev/null +++ b/concurrency/tests/job_core_loom.rs @@ -0,0 +1,640 @@ +//! Loom model of the pipe-to-self job core (`src/reply/job_core.rs`, +//! docs/design/deferred-replies.md ยง8, ยง9, `verification/tla/DeferredJobs.tla`): +//! admission and its rollback, keyed replacement of a completed job, emits +//! racing the terminal notification and cancellation, native completion +//! racing the owner's cleanup join, and the final destruction of a job whose +//! last strong reference comes from a `Weak` upgrade. +//! +//! `Arc`/`Weak` for jobs are modelled below (Loom has no `Weak`): the strong +//! count is a Loom atomic and upgrade is the same CAS loop as `std`, so the +//! thread that drops the last strong reference is explored. A job's final +//! destruction takes the job set's locks, which deadlocks (and Loom reports +//! it) if the destruction runs under one of them. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; + pub(crate) use loom::sync::{Arc, Condvar, Mutex, MutexGuard}; + pub(crate) type SlotOwner = Option<()>; + pub(crate) type Reason = (); + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + f() + } + pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} + pub(crate) fn shortened(_: &SlotOwner, _: usize) {} + pub(crate) fn parked() {} + pub(crate) use super::model_arc::{JobArc, JobWeak}; + + /// The thread's park token: `std::thread::park`'s semantics (a wake + /// sets the token; a park consumes it), modelled with a Loom mutex and + /// condvar because Loom's own `unpark` also wakes a thread blocked in a + /// `join`. + #[derive(Default)] + pub(crate) struct Token { + set: loom::sync::Mutex, + cv: loom::sync::Condvar, + } + impl std::task::Wake for Token { + fn wake(self: std::sync::Arc) { + *self.set.lock().unwrap() = true; + self.cv.notify_all(); + } + } + loom::thread_local! { + static TOKEN: std::sync::Arc = std::sync::Arc::new(Token::default()); + } + + /// A waker that unparks the current Loom thread. + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + let waker = TOKEN.with(|token| std::task::Waker::from(token.clone())); + f(&waker) + } + + /// Loom has no timed park; tests park only on slots without a deadline. + pub(crate) fn park(ticks: Option) { + assert!(ticks.is_none(), "timed park is not modelled"); + let token = TOKEN.with(|token| token.clone()); + let mut set = token.set.lock().unwrap(); + while !*set { + set = token.cv.wait(set).unwrap(); + } + *set = false; + } +} + +/// `std::sync::{Arc, Weak}` for jobs, with a hook at the final strong drop. +mod model_arc { + use loom::sync::atomic::{AtomicUsize, Ordering}; + use std::any::Any; + + struct Shared { + strong: AtomicUsize, + value: T, + } + + pub(crate) struct JobArc(loom::sync::Arc>); + pub(crate) struct JobWeak(loom::sync::Arc>); + + impl JobArc { + pub(crate) fn new(value: T) -> Self { + Self(loom::sync::Arc::new(Shared { + strong: AtomicUsize::new(1), + value, + })) + } + pub(crate) fn downgrade(this: &Self) -> JobWeak { + JobWeak(this.0.clone()) + } + } + impl Clone for JobArc { + fn clone(&self) -> Self { + self.0.strong.fetch_add(1, Ordering::Relaxed); + Self(self.0.clone()) + } + } + impl std::ops::Deref for JobArc { + type Target = T; + fn deref(&self) -> &T { + &self.0.value + } + } + impl Drop for JobArc { + fn drop(&mut self) { + if self.0.strong.fetch_sub(1, Ordering::AcqRel) == 1 { + if let Some(job) = (&self.0.value as &dyn Any).downcast_ref::() { + job.destroyed(); + } + } + } + } + impl JobWeak { + /// `std::sync::Weak::upgrade`: increment unless already zero. + pub(crate) fn upgrade(&self) -> Option> { + let mut n = self.0.strong.load(Ordering::Relaxed); + loop { + if n == 0 { + return None; + } + match self + .0 + .strong + .compare_exchange(n, n + 1, Ordering::Acquire, Ordering::Relaxed) + { + Ok(_) => return Some(JobArc(self.0.clone())), + Err(now) => n = now, + } + } + } + } +} + +#[allow(dead_code)] +#[path = "../src/reply/job_core.rs"] +mod job_core; + +#[allow(dead_code)] +#[path = "../src/reply/slot.rs"] +mod slot; + +use job_core::{Job, JobCore, JobGone, JobSetCore, Notification}; +use loom::sync::atomic::{AtomicUsize, Ordering}; +use loom::sync::{Arc, Mutex}; +use model_arc::JobArc; +use slot::CreditPool; +use std::sync::atomic::Ordering as StdOrdering; + +type Set = JobSetCore; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.check(f); +} + +/// A job: the real core plus a destruction probe. +struct TestJob { + id: u32, + core: JobCore, + /// Taken at the final destruction, which then takes the set's locks. + set: Mutex>>, + destroyed: Arc, + gone: std::sync::atomic::AtomicBool, +} + +impl TestJob { + fn new( + id: u32, + credits: u16, + set: Option<&Arc>, + destroyed: &Arc, + ) -> JobArc { + JobArc::new(Self { + id, + core: JobCore::new(credits), + set: Mutex::new(set.cloned()), + destroyed: destroyed.clone(), + gone: std::sync::atomic::AtomicBool::new(false), + }) + } + fn destroyed(&self) { + assert!( + !self.gone.swap(true, StdOrdering::AcqRel), + "job {} destroyed twice", + self.id + ); + self.destroyed.fetch_add(1, Ordering::AcqRel); + let set = self.set.lock().unwrap().take(); + if let Some(set) = set { + set.probe_locks(); + } + } +} + +impl Job for TestJob { + fn cancel(&self) { + self.core.cancel(); + } + fn is_running(&self) -> bool { + self.core.is_running() + } + fn finished(&self) { + self.core.finished(); + } +} + +/// Messages a job enqueues to its owner. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Msg { + Emit(u32, u32), + Terminal(u32), +} + +/// The production emit credit guard (`EmitCredit`): returns the credit when +/// dropped unless it travelled with an enqueued message. +struct CreditGuard(Option>); +impl Drop for CreditGuard { + fn drop(&mut self) { + if let Some(job) = self.0.take() { + job.core.return_emit_credit(); + } + } +} + +#[derive(Default)] +struct Mailbox(Mutex>); +impl Mailbox { + fn push(&self, msg: Msg) { + self.0.lock().unwrap().push(msg); + } + fn take(&self) -> Vec { + std::mem::take(&mut *self.0.lock().unwrap()) + } +} + +/// Admission at the bound: a blocking job reserves a credit then a native +/// slot, and rolls both back when its thread cannot be spawned, racing a +/// running job that ends (releasing its native slot and credit) and a second +/// admission. The bounds are never exceeded and nothing leaks. +#[test] +fn admission_rollback_vs_job_end() { + model(|| { + // Two credits: the running job holds one, so both admissions can + // reach the native check at once. + let pool = Arc::new(CreditPool::new(2)); + let set = Arc::new(Set::new(1)); + let holding = Arc::new(AtomicUsize::new(0)); + let destroyed = Arc::new(AtomicUsize::new(0)); + // A running job holds the credit and the native slot. + assert!(pool.reserve(1) && set.reserve_native()); + let running = TestJob::new(0, 1, None, &destroyed); + assert!(set.admit(&running, None)); + holding.fetch_add(1, Ordering::AcqRel); + + let ender = { + let (pool, set, holding) = (pool.clone(), set.clone(), holding.clone()); + loom::thread::spawn(move || { + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + pool.release(1); + set.ended(&running); + }) + }; + // Admission whose spawn fails: roll back what was reserved. + let admitter = { + let (pool, set, holding) = (pool.clone(), set.clone(), holding.clone()); + loom::thread::spawn(move || { + if pool.reserve(1) { + if set.reserve_native() { + assert_eq!( + holding.fetch_add(1, Ordering::AcqRel), + 0, + "native bound exceeded" + ); + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + } + pool.release(1); + } + }) + }; + // A second admission on this thread: admitted, or rolled back. + let mut second = None; + if pool.reserve(1) { + if set.reserve_native() { + assert_eq!( + holding.fetch_add(1, Ordering::AcqRel), + 0, + "native bound exceeded" + ); + let job = TestJob::new(1, 1, None, &destroyed); + assert!(set.admit(&job, None)); + second = Some(job); + } else { + pool.release(1); + } + } + ender.join().unwrap(); + admitter.join().unwrap(); + if let Some(job) = second { + holding.fetch_sub(1, Ordering::AcqRel); + set.release_native(); + pool.release(1); + set.ended(&job); + } + assert_eq!(set.running(), 0); + assert_eq!(set.native(), 0, "a native slot leaked"); + assert_eq!(pool.used(), 0, "a credit leaked"); + }); +} + +/// A keyed job completes (terminal message enqueued, execution ended) while +/// the owner replaces its key: the replaced job is cancelled whatever its +/// notification state, so its terminal message is fenced at dispatch. +#[test] +fn completed_key_replacement() { + model(|| { + let set = Arc::new(Set::new(1)); + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let old = TestJob::new(0, 1, None, &destroyed); + assert!(set.admit(&old, Some(7))); + let worker = { + let (set, mailbox, old) = (set.clone(), mailbox.clone(), old.clone()); + loom::thread::spawn(move || { + old.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + set.ended(&old); + }) + }; + let new = TestJob::new(1, 1, None, &destroyed); + assert!(set.admit(&new, Some(7))); + assert!( + old.core.is_cancelled(), + "the replaced job was not cancelled" + ); + assert!(!new.core.is_cancelled()); + worker.join().unwrap(); + // Dispatch after the replacement: the old terminal message is fenced. + for msg in mailbox.take() { + assert_eq!(msg, Msg::Terminal(0)); + assert!( + old.core.is_cancelled(), + "a replaced job's terminal message was dispatched" + ); + } + assert_ne!(old.core.notification(), Notification::Pending); + set.cancel_all(); + }); +} + +/// Admission races the owner's exit (review P1, Codex 6): `admit` checks +/// the gate and lists the job in the step `cancel_all` closes and drains +/// under, so a job is either refused or listed in time to be cancelled, and +/// the running count the owner's join waits for includes it. +#[test] +fn admission_vs_owner_exit() { + model(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + let admitter = { + let (set, job) = (set.clone(), job.clone()); + loom::thread::spawn(move || set.admit(&job, Some(1))) + }; + set.cancel_all(); + if admitter.join().unwrap() { + assert!( + job.core.is_cancelled(), + "an admitted job escaped the owner's exit" + ); + assert_eq!(set.running(), 1, "the owner's join misses the job"); + set.ended(&job); + } else { + assert_eq!(set.running(), 0); + } + let late = TestJob::new(1, 1, None, &destroyed); + assert!(!set.admit(&late, None), "admission reopened after exit"); + assert_eq!(set.running(), 0); + }); +} + +/// An emitter (one credit, two emits) races the terminal notification and, +/// optionally, a cancel. No emit is enqueued after the terminal message, a +/// blocked emitter is always released, and every credit comes back. +#[test] +fn emit_vs_notification_and_cancel() { + for cancel in [false, true] { + model(move || { + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + let emitter = { + let (job, mailbox) = (job.clone(), mailbox.clone()); + loom::thread::spawn(move || -> Result<(), JobGone> { + for i in 0..2 { + job.core.acquire_blocking()?; + // The production credit guard: it moves into the + // envelope when enqueued and returns the credit when + // dropped otherwise (the only owner of the return). + let credit = CreditGuard(Some(job.clone())); + job.core + .enqueue_emit(|| { + let mut credit = credit; + credit.0 = None; // travels with the message + mailbox.push(Msg::Emit(0, i)); + Ok::<(), ()>(()) + }) + .map_err(|(gone, _)| gone)?; + } + Ok(()) + }) + }; + let canceller = cancel.then(|| { + let job = job.clone(); + loom::thread::spawn(move || job.core.cancel()) + }); + let (taken, _) = job.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + if let Some(canceller) = canceller { + canceller.join().unwrap(); + } + // Dispatch: each emit's credit returns. + let mut terminal_seen = false; + for msg in mailbox.take() { + match msg { + Msg::Terminal(_) => terminal_seen = true, + Msg::Emit(..) => { + assert!( + !terminal_seen, + "an emit was enqueued after the terminal message" + ); + job.core.return_emit_credit(); + } + } + } + assert_eq!(terminal_seen, taken); + if !cancel { + assert!(taken, "nothing suppressed the notification"); + } + // The emitter stops at the terminal notification or the cancel + // (or after both emits); nothing is enqueued after the dispatch. + let _ = emitter.join().unwrap(); + for msg in mailbox.take() { + panic!("enqueued after dispatch: {msg:?}"); + } + assert_eq!(job.core.emit_credits(), 1, "an emit credit was lost"); + }); + } +} + +/// A delayed emit (`emit_at`) scheduled by the work fires on the deadline +/// service while the work finishes and, optionally, the job is cancelled. +/// The terminal message is held until the scheduled emit has fired, so it is +/// always last; it is delivered exactly once unless cancelled; a cancel drops +/// a held result; every credit comes back. +#[test] +fn scheduled_emit_vs_finish_and_cancel() { + for cancel in [false, true] { + model(move || { + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, None, &destroyed); + // The work schedules one delayed emit (its credit taken now). + job.core.acquire_blocking().unwrap(); + job.core.schedule().unwrap(); + let timer = { + let (job, mailbox) = (job.clone(), mailbox.clone()); + loom::thread::spawn(move || { + let (result, held, unused) = job.core.fire(Some(|| { + mailbox.push(Msg::Emit(0, 0)); + Ok::<(), ()>(()) + })); + // Not enqueued: handed back to drop outside the lock. + assert_eq!(result.is_err(), unused.is_some()); + if result.is_err() { + // Not enqueued: the credit guard returns it. + job.core.return_emit_credit(); + } + if let Some(held) = held { + held(); + } + }) + }; + let canceller = cancel.then(|| { + let job = job.clone(); + loom::thread::spawn(move || job.core.cancel()) + }); + // The work ends: notify now, or hold for the scheduled emit. + let later_job = job.clone(); + let later_mailbox = mailbox.clone(); + job.core.finish( + || { + mailbox.push(Msg::Terminal(0)); + None::<()> + }, + move || { + Box::new(move || { + later_job.core.notify(|| { + later_mailbox.push(Msg::Terminal(0)); + None::<()> + }); + }) + }, + ); + timer.join().unwrap(); + if let Some(canceller) = canceller { + canceller.join().unwrap(); + } + let (mut terminals, mut emits) = (0, 0); + for msg in mailbox.take() { + match msg { + Msg::Terminal(_) => terminals += 1, + Msg::Emit(..) => { + assert_eq!(terminals, 0, "the delayed emit followed the terminal"); + emits += 1; + job.core.return_emit_credit(); + } + } + } + assert!(terminals <= 1, "two terminal messages"); + assert_eq!(job.core.scheduled(), 0); + if !cancel { + assert_eq!(terminals, 1, "the held terminal was lost"); + assert_eq!(emits, 1, "the delayed emit was dropped without a cancel"); + } + if terminals == 0 { + assert_eq!(job.core.notification(), Notification::Suppressed); + } + assert_eq!(job.core.emit_credits(), 1, "an emit credit was lost"); + }); + } +} + +/// A native job completes (native slot released, terminal notification, +/// execution ended) while the owner exits: cancel every job, then join. +/// The join is always woken, and once it returns no job runs, the native +/// slot is free and the job's notification is decided. +#[test] +fn native_completion_vs_cleanup() { + model(|| { + let set = Arc::new(Set::new(1)); + let mailbox = Arc::new(Mailbox::default()); + let destroyed = Arc::new(AtomicUsize::new(0)); + assert!(set.reserve_native()); + let job = TestJob::new(0, 1, None, &destroyed); + set.admit(&job, None); + let native = { + let (set, mailbox, job) = (set.clone(), mailbox.clone(), job.clone()); + loom::thread::spawn(move || { + set.release_native(); + job.core.notify(|| { + mailbox.push(Msg::Terminal(0)); + None::<()> + }); + set.ended(&job); + }) + }; + set.cancel_all(); + let unfinished = set.join_with(|idle, running| Ok(idle.wait(running).unwrap())); + assert_eq!(unfinished, 0); + assert_eq!( + set.native(), + 0, + "the join returned before the native slot was released" + ); + assert!(!job.core.is_running()); + assert_ne!(job.core.notification(), Notification::Pending); + native.join().unwrap(); + }); +} + +/// The owner's cleanup upgrades its weak job list while the job thread drops +/// the last other strong reference: the job is destroyed exactly once, by +/// whichever thread drops last, and never under the set's locks. +#[test] +fn weak_upgrade_final_destruction_at_cancel_all() { + model(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let job = TestJob::new(0, 1, Some(&set), &destroyed); + set.admit(&job, Some(1)); + let worker = { + let set = set.clone(); + loom::thread::spawn(move || { + set.ended(&job); + drop(job); + }) + }; + set.cancel_all(); + worker.join().unwrap(); + assert_eq!( + destroyed.load(Ordering::Acquire), + 1, + "the job was not destroyed exactly once" + ); + }); +} + +/// Compaction of the weak job list (at 32 entries) upgrades a finished job +/// whose last other reference is being dropped: if the upgraded reference is +/// the last, the destruction runs after the lock is released. +#[test] +fn weak_upgrade_final_destruction_at_admission() { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(2); + builder.check(|| { + let set = Arc::new(Set::new(1)); + let destroyed = Arc::new(AtomicUsize::new(0)); + let filler = Arc::new(AtomicUsize::new(0)); + // 31 entries already dead, plus the racing job: 32 at admission. + for id in 0..31 { + let dead = TestJob::new(id, 1, None, &filler); + set.admit(&dead, None); + set.ended(&dead); + } + let job = TestJob::new(0, 1, Some(&set), &destroyed); + set.admit(&job, None); + let worker = { + let set = set.clone(); + loom::thread::spawn(move || { + set.ended(&job); + drop(job); + }) + }; + // Compacts the weak list, upgrading the racing job. + let next = TestJob::new(1, 1, None, &destroyed); + set.admit(&next, Some(1)); + worker.join().unwrap(); + assert_eq!( + destroyed.load(Ordering::Acquire), + 1, + "the job was not destroyed exactly once" + ); + set.ended(&next); + set.cancel_all(); + }); +} diff --git a/concurrency/tests/job_trace.rs b/concurrency/tests/job_trace.rs new file mode 100644 index 000000000..f42efa959 --- /dev/null +++ b/concurrency/tests/job_trace.rs @@ -0,0 +1,685 @@ +//! Records pipe-to-self job traces from real actors for TLA+ trace +//! validation against `verification/tla/DeferredJobs.tla` +//! (verification/tla/README.md). Each scenario writes `jobs_.json` to +//! `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `jobs-v1`). +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{ + Actor, ActorRef, ActorStart, Context, Handler, JobCtx, JobHandle, JobOptions, JobOutcome, + PendingReply, + }, +}; +use spawned_rt::tasks::{oneshot, sleep}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +/// The gated old keyed job and its gate, handed from the turn to the test. +type OldJob = Arc)>>>; + +#[derive(Default)] +struct Log(Mutex>); +impl Log { + fn push(&self, entry: impl Into) { + self.0.lock().unwrap().push(entry.into()); + } + fn snapshot(&self) -> Vec { + self.0.lock().unwrap().clone() + } +} + +struct Owner { + log: Arc, + handles: Vec, + gates: Vec>, + /// The gated old keyed job, handed to the test. + old: OldJob, +} + +impl Actor for Owner {} + +struct Done(String); +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(msg.0); + } +} + +struct Emitted(u32); +impl Message for Emitted { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Emitted, _: PendingReply<'_, ()>, _: &Context) { + self.log.push(format!("emit:{}", msg.0)); + } +} + +fn done(name: &'static str) -> impl FnOnce(JobOutcome) -> Done { + move |outcome| { + Done(match outcome { + Ok(v) => format!("{name}:ok:{v:?}"), + Err(e) => format!("{name}:{e}"), + }) + } +} + +#[derive(Clone, Copy)] +enum Scenario { + Plain, + KeyedFenced, + Emits, + Gated, + Deadline, + Blocking, + /// emit_at in the future, work done first: the result is held and the + /// last fire releases it. + HeldRelease, + /// emit_at already due: fired at the call. + DueNow, + /// The job deadline takes a held result; the later fire is dropped. + HeldDeadline, + /// A cancel drops the scheduled emit. + CancelScheduled, +} + +struct Run(Scenario); +impl Message for Run { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Run, reply: PendingReply<'_, ()>, ctx: &Context) { + match msg.0 { + Scenario::Plain => { + ctx.pipe(JobOptions::new(), |_| async { 1u32 }, done("plain")) + .unwrap(); + } + Scenario::KeyedFenced => { + let (open, gate) = oneshot::channel::<()>(); + let old = ctx + .pipe( + JobOptions::new().key("k"), + |_| async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + done("old"), + ) + .unwrap(); + *self.old.lock().unwrap() = Some((old, open)); + } + Scenario::Emits => { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe( + opts, + |job: JobCtx| async move { + for i in 0..3 { + if job.emit(Emitted(i)).await.is_err() { + break; + } + } + 3u32 + }, + done("emits"), + ) + .unwrap(); + } + Scenario::Gated => { + let (open, gate) = oneshot::channel::<()>(); + let handle = ctx + .pipe( + JobOptions::new(), + |_| async move { gate.await.map(|()| 1u32).unwrap_or(0) }, + done("gated"), + ) + .unwrap(); + self.handles.push(handle); + self.gates.push(open); + } + Scenario::Deadline => { + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe( + opts, + |_| async { + sleep(Duration::from_millis(120)).await; + 1u32 + }, + done("slow"), + ) + .unwrap(); + } + Scenario::Blocking => { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(2).unwrap()); + ctx.pipe_blocking( + opts, + |job: JobCtx| { + let _ = job.emit_blocking(Emitted(0)); + let _ = job.emit_blocking(Emitted(1)); + 5u32 + }, + done("blocking"), + ) + .unwrap(); + } + Scenario::HeldRelease => { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe( + opts, + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(60); + job.emit_at(at, Emitted(9)).await.unwrap(); + 7u32 + }, + done("held"), + ) + .unwrap(); + } + Scenario::DueNow => { + ctx.pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now(); + job.emit_at(at, Emitted(4)).await.unwrap(); + 4u32 + }, + done("due"), + ) + .unwrap(); + } + Scenario::HeldDeadline => { + let opts = JobOptions::new().deadline(Duration::from_millis(30)); + ctx.pipe( + opts, + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(150); + job.emit_at(at, Emitted(8)).await.unwrap(); + 8u32 + }, + done("held_dl"), + ) + .unwrap(); + } + Scenario::CancelScheduled => { + let handle = ctx + .pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(6)).await.unwrap(); + job.cancelled().await; + 6u32 + }, + done("cancel_sched"), + ) + .unwrap(); + self.handles.push(handle); + } + } + reply.reply(()); + } +} + +/// Hold the turn until `go` fires, then replace key `k`: the old job finished +/// meanwhile, so its terminal message is queued behind this turn and fenced. +/// The turn awaits instead of blocking the worker: a blocked worker strands +/// the old job, spawned into its non-stealable LIFO slot in the same poll. +struct ReplaceWhen(oneshot::Receiver<()>); +impl Message for ReplaceWhen { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: ReplaceWhen, _: PendingReply<'_, ()>, ctx: &Context) { + msg.0.await.unwrap(); + ctx.pipe(JobOptions::new().key("k"), |_| async { 2u32 }, done("new")) + .unwrap(); + } +} + +/// A keyed job that schedules an emit far away and waits to be cancelled; +/// `armed` fires once the emit is registered. +struct KeyedScheduled(oneshot::Sender<()>); +impl Message for KeyedScheduled { + type Result = (); +} +impl Handler for Owner { + async fn handle( + &mut self, + msg: KeyedScheduled, + reply: PendingReply<'_, ()>, + ctx: &Context, + ) { + let armed = msg.0; + ctx.pipe( + JobOptions::new().key("k"), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(5)).await.unwrap(); + let _ = armed.send(()); + job.cancelled().await; + 5u32 + }, + done("old_sched"), + ) + .unwrap(); + reply.reply(()); + } +} +/// Replace key `k` now. +struct ReplaceNow; +impl Message for ReplaceNow { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: ReplaceNow, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.pipe(JobOptions::new().key("k"), |_| async { 2u32 }, done("new")) + .unwrap(); + reply.reply(()); + } +} + +/// A held result whose destructor checks, from another thread, that the +/// process-wide trace section is free (it waits at most 1 s). +struct SectionProbe(Arc); +impl std::fmt::Debug for SectionProbe { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("SectionProbe") + } +} +impl Drop for SectionProbe { + fn drop(&mut self) { + let (entered_tx, entered) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + // Enters the trace section (no effect here: this binary does not + // record the base lifecycle). + let _ = lifecycle_trace::take_lifecycle(); + let _ = entered_tx.send(()); + }); + if entered.recv_timeout(Duration::from_secs(1)).is_ok() { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } + } +} +/// A job whose result (a `SectionProbe`) is held behind a far emit; its +/// handle goes to the test and to `handles` (for `CancelStored`). +struct HeldProbe( + Arc, + std::sync::mpsc::Sender, +); +impl Message for HeldProbe { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: HeldProbe, reply: PendingReply<'_, ()>, ctx: &Context) { + let HeldProbe(free, handle_tx) = msg; + let handle = ctx + .pipe( + JobOptions::new(), + |job: JobCtx| async move { + let at = spawned_concurrency::Instant::now() + Duration::from_secs(60); + job.emit_at(at, Emitted(3)).await.unwrap(); + SectionProbe(free) + }, + done("held_probe"), + ) + .unwrap(); + handle_tx.send(handle.clone()).unwrap(); + self.handles.push(handle); + reply.reply(()); + } +} + +/// Cancel the stored handles, then open the gates. +struct CancelStored; +impl Message for CancelStored { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: CancelStored, reply: PendingReply<'_, ()>, _: &Context) { + for handle in self.handles.drain(..) { + handle.cancel(); + } + for gate in self.gates.drain(..) { + let _ = gate.send(()); + } + reply.reply(()); + } +} + +fn start() -> (ActorRef, Arc) { + start_with(Arc::default()) +} + +fn start_with(old: OldJob) -> (ActorRef, Arc) { + let log = Arc::new(Log::default()); + let actor = Owner { + old, + log: log.clone(), + handles: Vec::new(), + gates: Vec::new(), + } + .start(); + lifecycle_trace::watch_deferred(actor.id().to_string().parse().unwrap()); + (actor, log) +} + +async fn wait_for(log: &Log, entry: &str) { + for _ in 0..400 { + if log.snapshot().iter().any(|e| e == entry) { + return; + } + sleep(Duration::from_millis(5)).await; + } + panic!("{entry} never logged: {:?}", log.snapshot()); +} + +/// Stop the owner, wait for its cleanup (the bounded job join), and write +/// its job trace. +async fn finish(name: &str, actor: ActorRef, credits: u16) -> Vec { + let id: u64 = actor.id().to_string().parse().unwrap(); + actor.context().stop(); + actor.join().await; + let events = lifecycle_trace::take_jobs(id); + assert!( + events.iter().any(|e| e.action == "Exit"), + "{name}: the owner's exit was not recorded" + ); + let actions: Vec = events.iter().map(|e| e.action.to_string()).collect(); + let json = format!( + "{{\"name\":\"jobs_{name}\",\"schema\":\"jobs-v1\",\"credits\":{credits},\"keys\":1,\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("jobs_{name}.json")), json).unwrap(); + actions +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn plain_job_result_is_dispatched() { + let (actor, log) = start(); + actor.request(Run(Scenario::Plain)).await.unwrap(); + wait_for(&log, "plain:ok:1").await; + let actions = finish("plain", actor, 16).await; + assert_eq!(actions, ["Start", "Finish", "Dispatch", "Exit"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn replaced_finished_keyed_job_is_fenced() { + let old = Arc::default(); + let (actor, log) = start_with(Arc::clone(&old)); + actor.request(Run(Scenario::KeyedFenced)).await.unwrap(); + let (go, held) = oneshot::channel::<()>(); + actor.send(ReplaceWhen(held)).unwrap(); + let (handle, open) = old.lock().unwrap().take().unwrap(); + open.send(()).unwrap(); + while handle.is_running() { + sleep(Duration::from_millis(1)).await; + } + go.send(()).unwrap(); + wait_for(&log, "new:ok:2").await; + finish("keyed_fenced", actor, 16).await; + assert!(!log.snapshot().iter().any(|e| e.starts_with("old"))); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn emits_precede_the_terminal_message() { + let (actor, log) = start(); + actor.request(Run(Scenario::Emits)).await.unwrap(); + wait_for(&log, "emits:ok:3").await; + finish("emits", actor, 1).await; + assert_eq!(log.snapshot(), ["emit:0", "emit:1", "emit:2", "emits:ok:3"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_job_is_quiet() { + let (actor, log) = start(); + actor.request(Run(Scenario::Gated)).await.unwrap(); + actor.request(CancelStored).await.unwrap(); + sleep(Duration::from_millis(30)).await; + finish("cancelled", actor, 16).await; + assert!(log.snapshot().is_empty(), "{:?}", log.snapshot()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn job_deadline_reports_while_work_runs() { + let (actor, log) = start(); + actor.request(Run(Scenario::Deadline)).await.unwrap(); + wait_for(&log, "slow:job deadline exceeded").await; + sleep(Duration::from_millis(200)).await; + let actions = finish("deadline", actor, 16).await; + assert_eq!( + actions, + ["Start", "JobDeadline", "Dispatch", "Finish", "Exit"] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn owner_exit_cancels_a_running_job() { + let (actor, _log) = start(); + actor.request(Run(Scenario::Gated)).await.unwrap(); + let actions = finish("exit_running", actor, 16).await; + assert_eq!(actions, ["Start", "Exit", "Finish"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn blocking_job_emits_then_completes() { + let (actor, log) = start(); + actor.request(Run(Scenario::Blocking)).await.unwrap(); + wait_for(&log, "blocking:ok:5").await; + finish("blocking", actor, 2).await; + assert_eq!(log.snapshot(), ["emit:0", "emit:1", "blocking:ok:5"]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn delayed_emit_releases_the_held_result() { + let (actor, log) = start(); + actor.request(Run(Scenario::HeldRelease)).await.unwrap(); + wait_for(&log, "held:ok:7").await; + let actions = finish("held_release", actor, 1).await; + assert_eq!(log.snapshot(), ["emit:9", "held:ok:7"]); + for a in ["Schedule", "Finish", "Fire", "Release"] { + assert!(actions.iter().any(|x| x == a), "{a} missing: {actions:?}"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn due_delayed_emit_fires_at_the_call() { + let (actor, log) = start(); + actor.request(Run(Scenario::DueNow)).await.unwrap(); + wait_for(&log, "due:ok:4").await; + let actions = finish("due_now", actor, 16).await; + assert_eq!(log.snapshot(), ["emit:4", "due:ok:4"]); + assert_eq!( + actions, + ["Start", "Schedule", "Fire", "Finish", "Dispatch", "Dispatch", "Exit"] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn job_deadline_takes_a_held_result() { + let (actor, log) = start(); + actor.request(Run(Scenario::HeldDeadline)).await.unwrap(); + wait_for(&log, "held_dl:job deadline exceeded").await; + sleep(Duration::from_millis(250)).await; + let actions = finish("held_deadline", actor, 16).await; + assert_eq!(log.snapshot(), ["held_dl:job deadline exceeded"]); + assert_eq!( + actions, + [ + "Start", + "Schedule", + "Finish", + "JobDeadline", + "Dispatch", + "Fire", + "Exit" + ] + ); +} + +/// Final review (agy P1 2): a keyed replacement cancels the old job in the +/// step of the new job's `Start` (the model's CancelCore(ToReplace)), so +/// the old job's dropped emit is recorded after that `Start`. The +/// replacement used to cancel first, so the drop preceded the `Start` and +/// TraceJobs rejected the trace; the old job's payloads also dropped inside +/// the trace section. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn keyed_replacement_drops_the_old_emit_after_the_new_start() { + let (actor, log) = start(); + let (armed_tx, armed) = oneshot::channel(); + actor.request(KeyedScheduled(armed_tx)).await.unwrap(); + armed.await.unwrap(); // the old job's emit is registered + actor.request(ReplaceNow).await.unwrap(); + wait_for(&log, "new:ok:2").await; + let actions = finish("keyed_replace_scheduled", actor, 16).await; + let second_start = actions + .iter() + .enumerate() + .filter(|(_, a)| *a == "Start") + .nth(1) + .map(|(i, _)| i) + .expect("the replacement's Start"); + let dropped_fire = actions + .iter() + .position(|a| a == "Fire") + .expect("the old job's dropped emit"); + assert!( + second_start < dropped_fire, + "the replaced job's emit dropped before the new Start: {actions:?}" + ); +} + +/// Final review (agy P2 3): cancelling a job drops its suppressed held +/// result (user payload) after the trace section, after its Cancel is +/// recorded; it used to drop inside the process-wide section, so a +/// destructor that needed the section (here, on another thread) waited for +/// its own caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_drops_a_held_result_outside_the_trace_section() { + let (actor, _log) = start(); + let free = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let (handle_tx, handle) = std::sync::mpsc::channel(); + actor + .request(HeldProbe(free.clone(), handle_tx)) + .await + .unwrap(); + let job = handle.recv_timeout(Duration::from_secs(5)).unwrap(); + // Handshake: the work returned with its emit scheduled, so its result + // is held. + while job.is_running() { + sleep(Duration::from_millis(1)).await; + } + actor.request(CancelStored).await.unwrap(); + finish("cancel_held", actor, 16).await; + assert!( + free.load(std::sync::atomic::Ordering::SeqCst), + "a held result was destroyed inside the trace section" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancel_drops_a_scheduled_emit() { + let (actor, log) = start(); + actor.request(Run(Scenario::CancelScheduled)).await.unwrap(); + // The emit is registered before the job awaits its cancellation. + sleep(Duration::from_millis(30)).await; + actor.request(CancelStored).await.unwrap(); + sleep(Duration::from_millis(30)).await; + let actions = finish("cancel_scheduled", actor, 16).await; + assert!(log.snapshot().is_empty(), "{:?}", log.snapshot()); + assert_eq!( + actions, + ["Start", "Schedule", "Cancel", "Fire", "Finish", "Exit"] + ); +} + +/// Threads mode: `emit_at_blocking` from a `pipe_blocking` job, result held +/// until the delayed emit fires. +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{ + Actor as TActor, ActorStart as _, Context as TContext, Handler as THandler, + JobCtx as TJobCtx, + }; + + struct TOwner(Arc); + impl TActor for TOwner {} + impl THandler for TOwner { + fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &TContext) { + self.0.push(msg.0); + } + } + impl THandler for TOwner { + fn handle(&mut self, msg: Emitted, _: PendingReply<'_, ()>, _: &TContext) { + self.0.push(format!("emit:{}", msg.0)); + } + } + struct Go; + impl Message for Go { + type Result = (); + } + impl THandler for TOwner { + fn handle(&mut self, _: Go, reply: PendingReply<'_, ()>, ctx: &TContext) { + let opts = JobOptions::new().emit_credits(std::num::NonZeroU16::new(1).unwrap()); + ctx.pipe_blocking( + opts, + |job: TJobCtx| { + let at = spawned_concurrency::Instant::now() + Duration::from_millis(60); + job.emit_at_blocking(at, Emitted(3)).unwrap(); + 3u32 + }, + done("theld"), + ) + .unwrap(); + reply.reply(()); + } + } + + #[test] + fn delayed_emit_releases_the_held_result_threads() { + let log = Arc::new(Log::default()); + let actor = TOwner(log.clone()).start(); + let id: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(id); + actor.request(Go).unwrap(); + for _ in 0..400 { + if log.snapshot().iter().any(|e| e == "theld:ok:3") { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(log.snapshot(), ["emit:3", "theld:ok:3"]); + actor.context().stop(); + actor.join(); + let events = lifecycle_trace::take_jobs(id); + let actions: Vec<&str> = events.iter().map(|e| e.action).collect(); + for a in ["Schedule", "Finish", "Fire", "Release", "Exit"] { + assert!(actions.contains(&a), "{a} missing: {actions:?}"); + } + let json = format!( + "{{\"name\":\"jobs_held_release_threads\",\"schema\":\"jobs-v1\",\"credits\":1,\"keys\":1,\"events\":[\n{}\n]}}\n", + events.iter().map(|e| e.to_json()).collect::>().join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("jobs_held_release_threads.json"), json).unwrap(); + } +} diff --git a/concurrency/tests/kill_links.rs b/concurrency/tests/kill_links.rs index 0b77a6db4..729347ead 100644 --- a/concurrency/tests/kill_links.rs +++ b/concurrency/tests/kill_links.rs @@ -22,8 +22,13 @@ fn native_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - fn handle(&mut self, _: Ping, _: &Context) -> u8 { - 7 + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); } } fn wait(actor: &ActorRef) -> ExitReason { @@ -78,8 +83,13 @@ async fn async_propagated_kill_is_trappable_but_direct_kill_is_not() { } } impl Handler for Node { - async fn handle(&mut self, _: Ping, _: &Context) -> u8 { - 7 + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); } } let (send, events) = mpsc::channel(); diff --git a/concurrency/tests/leaks.rs b/concurrency/tests/leaks.rs index b56c4cc82..db7ca83f8 100644 --- a/concurrency/tests/leaks.rs +++ b/concurrency/tests/leaks.rs @@ -67,7 +67,14 @@ impl threads::Actor for NativeProbe { } } impl threads::Handler for NativeProbe { - fn handle(&mut self, _: Ping, _: &threads::Context) {} + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &threads::Context, + ) { + reply.reply(()); + } } #[test] @@ -105,13 +112,25 @@ struct NativeB { impl threads::Actor for NativeA {} impl threads::Actor for NativeB {} impl threads::Handler>> for NativeA { - fn handle(&mut self, msg: SetPeer>, _: &threads::Context) { + fn handle( + &mut self, + msg: SetPeer>, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &threads::Context, + ) { self.peer = Some(msg.0); + reply.reply(()); } } impl threads::Handler>> for NativeB { - fn handle(&mut self, msg: SetPeer>, _: &threads::Context) { + fn handle( + &mut self, + msg: SetPeer>, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &threads::Context, + ) { self.peer = Some(msg.0); + reply.reply(()); } } @@ -202,19 +221,40 @@ mod async_leaks { } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } struct Crash; impl Message for Crash { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected failure for leak test"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected failure for leak test"); + } } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } // Not simulated: counts Tokio runtime tasks and OS threads. @@ -302,29 +342,47 @@ mod async_leaks { impl Actor for AsyncA {} impl Actor for AsyncB {} impl Handler for AsyncA { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for AsyncB { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler, tasks::Recipient)>> for AsyncA { async fn handle( &mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for AsyncB { async fn handle( &mut self, msg: SetPeer<(ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0 .0); self.recipient = Some(msg.0 .1); + reply.reply(()); } } @@ -425,16 +483,14 @@ mod async_leaks { let system = ActorSystem::new().unwrap(); let group = system.async_group("leaks", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let mut children = Vec::new(); for name in names { @@ -572,10 +628,24 @@ mod sim_leaks { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, _: Down, _: &Context) {} + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } /// A holds B's ref and recipient, B holds A's: a reference cycle through @@ -591,27 +661,45 @@ mod sim_leaks { impl Actor for A {} impl Actor for B {} impl Handler for A { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for B { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler, tasks::Recipient)>> for A { async fn handle( &mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0); + reply.reply(()); } } impl Handler, tasks::Recipient)>> for B { async fn handle( &mut self, msg: SetPeer<(tasks::ActorRef, tasks::Recipient)>, + reply: spawned_concurrency::PendingReply<'_, ()>, _: &Context, ) { self.peer = Some(msg.0); + reply.reply(()); } } @@ -742,7 +830,12 @@ mod sim_leaks { type Result = (); } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected failure for leak test"); } } @@ -793,16 +886,14 @@ mod sim_leaks { let system = ActorSystem::new().unwrap(); let group = system.async_group("leaks-sim-sup", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 100, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(100).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new(name, &group, move |_| { @@ -854,7 +945,12 @@ mod sim_leaks { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected failure for leak test"); } } diff --git a/concurrency/tests/lifecycle_trace.rs b/concurrency/tests/lifecycle_trace.rs new file mode 100644 index 000000000..51264dc4e --- /dev/null +++ b/concurrency/tests/lifecycle_trace.rs @@ -0,0 +1,811 @@ +//! Records base-lifecycle traces (exit signals, links, monitors, requests, +//! the death point and publication) from real actors for TLA+ trace +//! validation against `verification/tla/Lifecycle.tla` (mapping in +//! `TraceLifecycle.tla`). Each scenario writes `lifecycle_.json` to +//! `$LIFECYCLE_TRACE_DIR` (default `target/lifecycle-traces`); +//! `verification/tla/trace_check.py` checks them (schema `lifecycle-v1`). +//! +//! Recording is process-wide, so the scenarios run one at a time. +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + tasks::{Actor, ActorRef, ActorStart, Backend, Context, Handler}, + ActorError, ChildHandle, Down, Exit, ExitReason, MonitorRef, PendingReply, +}; +use std::{ + path::PathBuf, + sync::{Arc, Mutex}, + time::Duration, +}; + +const LIMIT: Duration = Duration::from_secs(10); + +/// Held for a whole scenario (also across awaits, so an async mutex). +static SERIAL: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +/// Stop recording and write the trace; returns the events as +/// `action[:x]` of the actor numbered by spawn order (`a1`, `a2`, ...). +fn write(name: &str) -> Vec { + let events = lifecycle_trace::take_lifecycle(); + let json = format!( + "{{\"name\":\"lifecycle_{name}\",\"schema\":\"lifecycle-v1\",\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("lifecycle_{name}.json")), json).unwrap(); + let actors: Vec = events + .iter() + .filter(|e| e.action == "Spawn") + .map(|e| e.t) + .collect(); + let index = |id: u64| actors.iter().position(|a| *a == id).map_or(0, |i| i + 1); + events + .iter() + .filter(|e| e.action != "Spawn") + .map(|e| { + let x = if e.x.is_empty() { + String::new() + } else { + format!(":{}", e.x) + }; + format!("a{}.{}{x}", index(e.t), e.action) + }) + .collect() +} + +fn has(events: &[String], wanted: &[&str]) { + // `wanted` must appear in this order (other events may interleave). + let mut at = 0; + for w in wanted { + match events[at..].iter().position(|e| e == w) { + Some(i) => at += i + 1, + None => panic!("{w} missing (in order) from {events:#?}"), + } + } +} + +type Log = Arc>>; + +fn logged(log: &Log) -> Vec { + log.lock().unwrap().clone() +} + +async fn wait_for(log: &Log, entry: &str) { + spawned_rt::tasks::timeout(LIMIT, async { + while !log.lock().unwrap().iter().any(|e| e == entry) { + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("{entry} never logged: {:?}", logged(log))); +} + +struct Node { + trap: bool, + log: Log, + /// `stopped()` waits for it. + hold_stopped: Option>, + /// The state destructor waits for it (a native thread: Backend::Thread). + hold_drop: Option>, +} + +impl Node { + fn new(log: &Log) -> Self { + Self { + trap: false, + log: log.clone(), + hold_stopped: None, + hold_drop: None, + } + } + fn trapping(log: &Log) -> Self { + let mut node = Self::new(log); + node.trap = true; + node + } + fn holding_stopped(log: &Log, gate: tokio::sync::oneshot::Receiver<()>) -> Self { + let mut node = Self::new(log); + node.hold_stopped = Some(gate); + node + } + fn holding_drop(gate: std::sync::mpsc::Receiver<()>) -> Self { + let mut node = Self::new(&Log::default()); + node.hold_drop = Some(gate); + node + } + fn note(&self, entry: impl Into) { + self.log.lock().unwrap().push(entry.into()); + } +} + +impl Drop for Node { + fn drop(&mut self) { + if let Some(gate) = self.hold_drop.take() { + let _ = gate.recv_timeout(LIMIT); + } + } +} + +impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, _: &Context) { + self.note("stopped"); + if let Some(gate) = self.hold_stopped.take() { + let _ = gate.await; + } + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.note(format!("exit:{:?}", exit.reason)); + } +} + +impl Handler for Node { + async fn handle(&mut self, down: Down, _: PendingReply<'_, ()>, _: &Context) { + self.note(format!("down:{:?}", down.reason)); + } +} + +macro_rules! request { + ($name:ident ( $($field:ty),* ) -> $result:ty, |$this:ident, $msg:ident, $ctx:ident| $body:expr) => { + struct $name($(pub $field),*); + impl Message for $name { + type Result = $result; + } + impl Handler<$name> for Node { + async fn handle( + &mut self, + $msg: $name, + reply: PendingReply<'_, $result>, + $ctx: &Context, + ) { + let $this = self; + let result = $body; + reply.reply(result); + } + } + }; +} + +request!(Sync() -> (), |_this, _msg, _ctx| ()); +request!(Link(ChildHandle) -> (), |_this, msg, ctx| ctx.link(&msg.0)); +request!(TryLink(ChildHandle) -> bool, |_this, msg, ctx| ctx.try_link(&msg.0).is_ok()); +request!(Unlink(ChildHandle) -> (), |_this, msg, ctx| ctx.unlink(&msg.0)); +request!(Monitor(ChildHandle) -> MonitorRef, |_this, msg, ctx| ctx.monitor(&msg.0)); +request!(Demonitor(MonitorRef) -> (), |_this, msg, ctx| ctx.demonitor(msg.0)); +request!(Note() -> (), |this, _msg, _ctx| this.note("note")); +request!(NoteThenExit(ActorRef) -> (), |_this, msg, ctx| { + msg.0.send(Note()).unwrap(); + ctx.stop_with_reason(ExitReason::Shutdown); +}); + +/// A request whose handler panics. +struct Boom(); +impl Message for Boom { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, _: Boom, _: PendingReply<'_, ()>, _: &Context) { + panic!("boom"); + } +} + +/// A request whose handler waits for a gate. +struct Block(tokio::sync::oneshot::Receiver<()>); +impl Message for Block { + type Result = (); +} +impl Handler for Node { + async fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + self.note("blocked"); + let _ = msg.0.await; + reply.reply(()); + } +} + +async fn exit_of(actor: &ActorRef) -> ExitReason { + spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) + .await + .expect("actor did not exit") +} + +/// A kill ends a running actor `Killed` without `stopped()`; the request +/// queued behind the current handler is answered at D with `Killed`, and a +/// request after D gets `NoProc`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn kill_answers_queued_request_at_death() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let (release, gate) = tokio::sync::oneshot::channel(); + let blocked = a1 + .send_request(Block(gate), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + wait_for(&log, "blocked").await; + let queued = a1 + .send_request(Sync(), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + a1.child_handle().kill(); + release.send(()).unwrap(); + assert!(blocked.await.is_ok()); + assert!(matches!( + queued.await, + Err(ActorError::ActorStopped(ExitReason::Killed)) + )); + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + assert!(matches!(a1.request(Sync()).await, Err(ActorError::NoProc))); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("kill_queued_request"); + has( + &events, + &[ + "a1.Dispatch", + "a1.Send:queued", + "a1.Signal:Kill", + "a1.Hook:skip", + "a1.Die:Killed", + "a1.Publish", + "a1.Send:noproc", + ], + ); +} + +/// A kill during a running `stopped()` replaces the requested reason; the +/// `stopped()` that began finishes. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn kill_during_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let (release, gate) = tokio::sync::oneshot::channel(); + let a1 = Node::holding_stopped(&log, gate).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().stop(); + wait_for(&log, "stopped").await; + a1.child_handle().kill(); + release.send(()).unwrap(); + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + let events = write("kill_during_stopped"); + has( + &events, + &[ + "a1.Stop:Normal", + "a1.Hook:run", + "a1.Signal:Kill", + "a1.Die:Killed", + "a1.Publish", + ], + ); +} + +/// An untrapped non-Normal exit signal ends the actor with its reason and +/// skips `stopped()`; a Normal one is ignored. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn untrapped_signal_skips_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Normal); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + assert_eq!(exit_of(&a1).await, ExitReason::Shutdown); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("untrapped_signal"); + has( + &events, + &[ + "a1.Signal:Normal", + "a1.Dispatch", + "a1.Signal:Shutdown", + "a1.Hook:skip", + "a1.Die:Shutdown", + "a1.Publish", + ], + ); +} + +/// A trapping actor gets exit signals as `Exit` messages, from outside and +/// from another actor, then stops normally through `stopped()`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn trapped_exit_reaches_exit_received() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a2.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + a2.context().exit(&a1.child_handle(), ExitReason::Kill); + // The second signal was a kill: untrappable, a1 ends Killed. + assert_eq!(exit_of(&a1).await, ExitReason::Killed); + a2.child_handle().stop(); + assert_eq!(exit_of(&a2).await, ExitReason::Normal); + let events = write("trapped_exit"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Signal:Kill", + "a1.Die:Killed", + "a2.Stop:Normal", + "a2.Hook:run", + "a2.Die:Normal", + ], + ); +} + +/// `ctx.exit` to a trapping actor queues an `Exit` that `exit_received` +/// handles; the actor keeps running until stopped. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn trapped_exit_then_stop() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + a1.request(Sync()).await.unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + assert!(logged(&log).contains(&"stopped".to_string())); + let events = write("trapped_exit_then_stop"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Stop:Normal", + "a1.Hook:run", + "a1.Die:Normal", + "a1.Publish", + ], + ); +} + +/// Link, try_link and monitor against an actor past D: a trapping caller +/// gets `Exit{NoProc}`, `try_link` returns `Err` and keeps nothing, the +/// monitor delivers `Down{NoProc}`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn late_registrations_get_noproc() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a2.child_handle().stop(); + assert_eq!(exit_of(&a2).await, ExitReason::Normal); + a1.request(Link(a2.child_handle())).await.unwrap(); + assert!(!a1.request(TryLink(a2.child_handle())).await.unwrap()); + a1.request(Monitor(a2.child_handle())).await.unwrap(); + wait_for(&log, "exit:NoProc").await; + wait_for(&log, "down:NoProc").await; + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + let events = write("late_registrations"); + has( + &events, + &[ + "a2.Die:Normal", + "a2.Publish", + "a1.Link:noproc", + "a1.Deliver:NoProc", + "a1.Link:try", + "a1.Monitor:late", + "a1.Deliver:NoProc", + "a1.Die:Normal", + ], + ); +} + +/// A non-trapping caller of link to an actor past D exits `NoProc` +/// without `stopped()`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn late_link_ends_a_non_trapping_caller() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a2.child_handle().stop(); + exit_of(&a2).await; + a1.request(Link(a2.child_handle())).await.unwrap(); + assert_eq!(exit_of(&a1).await, ExitReason::NoProc); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("late_link_untrapped"); + has( + &events, + &[ + "a1.Link:noproc", + "a1.Deliver:NoProc", + "a1.Hook:skip", + "a1.Die:NoProc", + ], + ); +} + +/// Unlink between the peer's D and its publication cancels the notice taken +/// at D (E6): no `Exit` reaches the trapping caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn unlink_fences_a_notice_taken_at_death() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let (release, gate) = std::sync::mpsc::channel(); + let a2 = Node::holding_drop(gate).start_with_backend(Backend::Thread); + a1.request(Link(a2.child_handle())).await.unwrap(); + a2.request(Sync()).await.unwrap(); + a2.context().stop_with_reason(ExitReason::Shutdown); + // Past D (requests get NoProc), destruction held: not yet published. + past_death(&a2).await; + a1.request(Unlink(a2.child_handle())).await.unwrap(); + release.send(()).unwrap(); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + a1.request(Sync()).await.unwrap(); + assert!( + !logged(&log).iter().any(|e| e.starts_with("exit:")), + "{:?}", + logged(&log) + ); + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("unlink_fence"); + has( + &events, + &[ + "a1.Link:linked", + "a2.Die:Shutdown", + "a1.Unlink", + "a2.Publish", + "a1.Die:Normal", + ], + ); + assert!(!events.iter().any(|e| e.starts_with("a1.Deliver"))); +} + +/// Demonitor between the target's D and its publication: no `Down`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn demonitor_fences_a_down() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let (release, gate) = std::sync::mpsc::channel(); + let a2 = Node::holding_drop(gate).start_with_backend(Backend::Thread); + let monitor = a1.request(Monitor(a2.child_handle())).await.unwrap(); + a2.child_handle().stop(); + past_death(&a2).await; + a1.request(Demonitor(monitor)).await.unwrap(); + release.send(()).unwrap(); + exit_of(&a2).await; + spawned_rt::tasks::sleep(Duration::from_millis(20)).await; + a1.request(Sync()).await.unwrap(); + assert!( + !logged(&log).iter().any(|e| e.starts_with("down:")), + "{:?}", + logged(&log) + ); + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("demonitor_fence"); + has( + &events, + &[ + "a1.Monitor:active", + "a2.Die:Normal", + "a1.Demonitor", + "a2.Publish", + "a1.Die:Normal", + ], + ); + assert!(!events.iter().any(|e| e.starts_with("a1.Deliver"))); +} + +async fn past_death(actor: &ActorRef) { + spawned_rt::tasks::timeout(LIMIT, async { + while !matches!(actor.request(Sync()).await, Err(ActorError::NoProc)) { + spawned_rt::tasks::sleep(Duration::from_millis(2)).await; + } + }) + .await + .expect("never reached its death point"); +} + +/// A monitor of a running actor delivers `Down` with the final reason after +/// publication. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn monitor_delivers_the_final_reason() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Monitor(a2.child_handle())).await.unwrap(); + a2.child_handle().stop_with_reason(ExitReason::Shutdown); + wait_for(&log, "down:Shutdown").await; + a1.child_handle().stop(); + exit_of(&a1).await; + let events = write("monitor_down"); + has( + &events, + &[ + "a1.Monitor:active", + "a2.Stop:Shutdown", + "a2.Die:Shutdown", + "a2.Publish", + "a1.Deliver:Shutdown", + "a1.Dispatch", + "a1.Die:Normal", + ], + ); +} + +/// The `start_linked` parent's exit ends a trapping child normally: a +/// parent exit, `stopped()` runs, and the child ends with the parent's +/// reason (E3). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn parent_exit_ends_a_trapping_child_through_stopped() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&Log::default()).start(); + a1.request(Sync()).await.unwrap(); + let a2 = Node::trapping(&log).start_linked(&a1.context()); + a2.request(Sync()).await.unwrap(); + a1.child_handle().stop_with_reason(ExitReason::Shutdown); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + assert!(logged(&log).contains(&"stopped".to_string())); + assert!(!logged(&log).iter().any(|e| e.starts_with("exit:"))); + let events = write("parent_exit"); + has( + &events, + &[ + "a2.Trap:true", + "a1.Stop:Shutdown", + "a1.Die:Shutdown", + "a1.Publish", + "a2.Deliver:Shutdown", + "a2.Dispatch", + "a2.Hook:run", + "a2.Die:Shutdown", + ], + ); +} + +/// A linked actor's panic ends a non-trapping peer with `Panic`, without +/// `stopped()`; the panicking request is answered at D with `Panic`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn link_notice_ends_a_non_trapping_peer() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::new(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Link(a2.child_handle())).await.unwrap(); + assert!(matches!( + a2.request(Boom()).await, + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); + assert!(matches!(exit_of(&a1).await, ExitReason::Panic(_))); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("linked_panic"); + has( + &events, + &[ + "a1.Link:linked", + "a2.Dispatch", + "a2.Panic:turn", + "a2.Hook:run", + "a2.Die:Panic", + "a2.Publish", + "a1.Deliver:Panic", + "a1.Hook:skip", + "a1.Die:Panic", + ], + ); +} + +/// The runtime drops a running actor: it ends `Killed`, no `stopped()` (G1). +#[test] +fn runtime_drop_ends_killed() { + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let a1 = runtime.block_on(async { + let a1 = Node::new(&log).start(); + a1.request(Sync()).await.unwrap(); + a1 + }); + drop(runtime); + assert_eq!(a1.child_handle().wait_exit_blocking(), ExitReason::Killed); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("runtime_drop"); + has(&events, &["a1.Dispatch", "a1.Drop:Killed", "a1.Publish"]); +} + +/// Threads mode: a trapped exit, then a kill skipping `stopped()`. +#[test] +fn threads_trapped_exit_then_kill() { + use spawned_concurrency::threads; + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + struct Native(Log); + impl threads::Actor for Native { + fn started(&mut self, ctx: &threads::Context) { + ctx.trap_exit(true); + } + fn stopped(&mut self, _: &threads::Context) { + self.0.lock().unwrap().push("stopped".into()); + } + fn exit_received(&mut self, exit: Exit, _: &threads::Context) { + self.0 + .lock() + .unwrap() + .push(format!("exit:{:?}", exit.reason)); + } + } + struct Ping; + impl Message for Ping { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &threads::Context) { + reply.reply(()); + } + } + use threads::ActorStart as _; + let log = Log::default(); + let a1 = Native(log.clone()).start(); + a1.request(Ping).unwrap(); + a1.child_handle().exit(ExitReason::Shutdown); + a1.request(Ping).unwrap(); + assert_eq!(logged(&log), ["exit:Shutdown"]); + a1.child_handle().kill(); + assert_eq!(a1.child_handle().wait_exit_blocking(), ExitReason::Killed); + assert!(!logged(&log).contains(&"stopped".to_string())); + let events = write("threads_trapped_exit_kill"); + has( + &events, + &[ + "a1.Trap:true", + "a1.Signal:Shutdown", + "a1.Dispatch", + "a1.Signal:Kill", + "a1.Hook:skip", + "a1.Die:Killed", + "a1.Publish", + ], + ); +} + +/// An actor-to-actor message (the model's `Send`) and the sender's death +/// notice after it (`ExitAfterMessages`): a2, linked to the trapping a1, +/// sends a1 a message in a turn and then stops `Shutdown`; a1 handles the +/// message before the `Exit`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn message_then_death_notice() { + let _serial = SERIAL.lock().await; + lifecycle_trace::record_lifecycle(); + let log = Log::default(); + let a1 = Node::trapping(&log).start(); + let a2 = Node::new(&Log::default()).start(); + a1.request(Sync()).await.unwrap(); + a2.request(Link(a1.child_handle())).await.unwrap(); + a2.request(NoteThenExit(a1.clone())).await.unwrap(); + assert_eq!(exit_of(&a2).await, ExitReason::Shutdown); + wait_for(&log, "exit:Shutdown").await; + assert_eq!(logged(&log), ["note", "exit:Shutdown"]); + a1.child_handle().stop(); + assert_eq!(exit_of(&a1).await, ExitReason::Normal); + let (id1, id2) = (a1.id().to_string(), a2.id().to_string()); + let events = write("message_then_exit"); + has( + &events, + &[ + "a1.Send:queued", + "a2.Stop:Shutdown", + "a2.Die:Shutdown", + "a1.Dispatch", + "a1.Dispatch", + ], + ); + let json = + std::fs::read_to_string(trace_dir().join("lifecycle_message_then_exit.json")).unwrap(); + assert!( + json.contains(&format!(r#""a":"Send","t":{id1},"s":{id2},"x":"queued""#)), + "the message was not recorded as a2's: {json}" + ); +} + +/// Threads mode: an actor-to-actor message is recorded as the sender's. +#[test] +fn threads_actor_message() { + use spawned_concurrency::threads; + let _serial = SERIAL.blocking_lock(); + lifecycle_trace::record_lifecycle(); + struct Native(Log); + impl threads::Actor for Native {} + struct Hello; + impl Message for Hello { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Hello, _: PendingReply<'_, ()>, _: &threads::Context) { + self.0.lock().unwrap().push("hello".into()); + } + } + struct Forward(threads::ActorRef); + impl Message for Forward { + type Result = (); + } + impl threads::Handler for Native { + fn handle( + &mut self, + msg: Forward, + reply: PendingReply<'_, ()>, + _: &threads::Context, + ) { + msg.0.send(Hello).unwrap(); + reply.reply(()); + } + } + struct Ping; + impl Message for Ping { + type Result = (); + } + impl threads::Handler for Native { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, ()>, _: &threads::Context) { + reply.reply(()); + } + } + use threads::ActorStart as _; + let log = Log::default(); + let a1 = Native(log.clone()).start(); + let a2 = Native(Log::default()).start(); + a2.request(Forward(a1.clone())).unwrap(); + a1.request(Ping).unwrap(); + assert_eq!(logged(&log), ["hello"]); + a1.child_handle().stop(); + a2.child_handle().stop(); + a1.child_handle().wait_exit_blocking(); + a2.child_handle().wait_exit_blocking(); + let (id1, id2) = (a1.id().to_string(), a2.id().to_string()); + let events = write("threads_actor_message"); + has(&events, &["a1.Send:queued", "a1.Dispatch"]); + let json = + std::fs::read_to_string(trace_dir().join("lifecycle_threads_actor_message.json")).unwrap(); + assert!( + json.contains(&format!(r#""a":"Send","t":{id1},"s":{id2},"x":"queued""#)), + "the message was not recorded as a2's: {json}" + ); +} diff --git a/concurrency/tests/link_fence_loom.rs b/concurrency/tests/link_fence_loom.rs new file mode 100644 index 000000000..acedd876a --- /dev/null +++ b/concurrency/tests/link_fence_loom.rs @@ -0,0 +1,548 @@ +//! Loom model of the link table and the monitor fence +//! (docs/design/otp-exit-signals.md E4-E6). The production protocols +//! (`src/link/table.rs`, `src/monitor/fence.rs`) are compiled here with +//! Loom's `Mutex`; `ActorId`, `ExitReason` and `SignalTarget` are stand-ins. +//! Applying a notice (production: `deliver_signal` on the receiver's exit +//! request, under the receiver's table lock) is modelled as appending to the +//! receiver's log under its own lock. Table `Arc`/`Weak` are `std` (only the +//! locks are explored; reference counts are not part of the protocol). +//! +//! Properties: +//! - at most one notice per edge generation, and none after the receiver's +//! `unlink` returned or after the receiver's own death point; +//! - a `link` racing the peer's death point gets exactly one notice: the +//! real reason (linked first) or a late `NoProc` (linked after D), never +//! both and never none; `try_link` after D gets none; +//! - at most one `Down` per monitor, none after `demonitor` returned or after +//! the observer's own close; a monitor racing the target's D gets exactly +//! one `Down`: the real reason or `NoProc`. +//! +//! Run: `RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test --release +//! -p spawned-concurrency --no-default-features --test link_fence_loom`. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::{Mutex, MutexGuard}; + pub(crate) use std::sync::{Arc, Weak}; +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub(crate) struct ActorId(u64); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum ExitReason { + Panic, + Shutdown, + NoProc, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct Notice { + from: ActorId, + gen: u64, + reason: ExitReason, +} + +/// The receiver side of a notice: its applied notices and fenced edges. +pub(crate) struct SignalTarget { + id: ActorId, + log: loom::sync::Mutex>, + /// Generations whose `unlink` (or the receiver's death) has returned. + fenced: loom::sync::Mutex>, +} + +#[allow(dead_code)] +#[path = "../src/link/table.rs"] +mod table; + +#[allow(dead_code)] +#[path = "../src/monitor/fence.rs"] +mod fence; + +use loom::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicUsize, Ordering as StdOrdering}; +use std::sync::Arc; +use table::{LinkOutcome, LinkTable}; + +fn model(name: &'static str, f: impl Fn() + Sync + Send + 'static) { + let runs = Arc::new(AtomicUsize::new(0)); + let counted = runs.clone(); + // Unbounded: the protocols are lock-based and small enough to explore + // every interleaving. + let builder = loom::model::Builder::new(); + builder.check(move || { + counted.fetch_add(1, StdOrdering::Relaxed); + f() + }); + eprintln!("{name}: {} executions", runs.load(StdOrdering::Relaxed)); +} + +struct Actor { + target: Arc, + links: LinkTable, +} + +impl Actor { + fn new(id: u64) -> Self { + Self { + target: Arc::new(SignalTarget { + id: ActorId(id), + log: loom::sync::Mutex::new(Vec::new()), + fenced: loom::sync::Mutex::new(Vec::new()), + }), + links: table::new_link_table(), + } + } + fn id(&self) -> ActorId { + self.target.id + } + fn log(&self) -> Vec { + self.target.log.lock().unwrap().clone() + } + fn fence(&self, gen: u64) { + self.target.fenced.lock().unwrap().push(gen); + } + fn records(&self) -> usize { + table::lock(&self.links).entries.len() + } +} + +/// `deliver_notice`: the receiver-side fenced step. +fn notice(links: &LinkTable, receiver: &SignalTarget, from: ActorId, gen: u64, reason: ExitReason) { + drop(table::take_record(links, from, gen, |record| { + assert_eq!(record.late, reason == ExitReason::NoProc); + assert!( + !receiver.fenced.lock().unwrap().contains(&gen), + "notice of edge {gen} took effect after unlink or death returned" + ); + let mut log = receiver.log.lock().unwrap(); + assert!( + log.iter().all(|n| n.gen != gen), + "a second notice of edge {gen}" + ); + log.push(Notice { from, gen, reason }); + })); +} + +/// `link_in_section` for a running caller. +fn link(own: &Actor, peer: &Actor, gen: u64, try_only: bool) -> LinkOutcome { + let outcome = table::install( + &own.target, + &own.links, + &peer.target, + &peer.links, + gen, + try_only, + ); + if outcome == LinkOutcome::NoProc && !try_only { + notice(&own.links, &own.target, peer.id(), gen, ExitReason::NoProc); + } + outcome +} + +/// `unregister_link`, then the edge is fenced on both sides. +fn unlink(own: &Actor, peer: &Actor, gen: u64) { + drop(table::remove_pair( + own.id(), + &own.links, + peer.id(), + &peer.links, + )); + own.fence(gen); +} + +/// The death point D, then publication (the notices taken at D). +fn die(actor: &Actor, reason: ExitReason, fenced: &[u64]) { + let notices = table::close(actor.id(), &actor.links, &reason, |_| {}); + // Past D nothing takes effect on this actor: its records are gone. + for gen in fenced { + actor.fence(*gen); + } + notices.deliver_with(notice); +} + +fn spawn(f: impl FnOnce() -> T + Send + 'static) -> loom::thread::JoinHandle { + loom::thread::spawn(f) +} + +/// `unlink` racing the peer's death point and publication: the notice +/// takes effect only before `unlink` returned, at most once. +#[test] +fn unlink_fences_a_death_notice() { + model("unlink_fences_a_death_notice", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + unlink(&b, &a, 1); + dying.join().unwrap(); + assert!(b.log().len() <= 1); + assert!(a.log().is_empty()); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// `link` racing the peer's death point: exactly one notice, the real +/// reason or a late `NoProc`; `try_link` after D gets none and no record. +#[test] +fn link_racing_death_gets_exactly_one_notice() { + for try_only in [false, true] { + model("link_racing_death_gets_exactly_one_notice", move || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Shutdown, &[])) + }; + let outcome = link(&b, &a, 1, try_only); + dying.join().unwrap(); + let log = b.log(); + match outcome { + LinkOutcome::Linked => assert_eq!( + log, + vec![Notice { + from: a.id(), + gen: 1, + reason: ExitReason::Shutdown + }] + ), + LinkOutcome::NoProc if try_only => assert!(log.is_empty()), + LinkOutcome::NoProc => assert_eq!( + log, + vec![Notice { + from: a.id(), + gen: 1, + reason: ExitReason::NoProc + }] + ), + LinkOutcome::Unchanged => panic!("a first link is never unchanged"), + } + assert_eq!(a.records() + b.records(), 0); + }); + } +} + +/// Unlink then relink while the old edge's notice is in flight: the old +/// generation never takes effect after the unlink; the new one gets exactly +/// one notice (the real reason, or `NoProc` past D). +#[test] +fn relink_after_unlink_ignores_the_stale_generation() { + model("relink_after_unlink_ignores_the_stale_generation", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + unlink(&b, &a, 1); + let outcome = link(&b, &a, 2, false); + dying.join().unwrap(); + let log = b.log(); + assert!(log.iter().filter(|n| n.gen == 1).count() <= 1); + let fresh: Vec<_> = log.iter().filter(|n| n.gen == 2).collect(); + assert_eq!(fresh.len(), 1, "{outcome:?}: {log:?}"); + let expected = match outcome { + LinkOutcome::Linked => ExitReason::Panic, + LinkOutcome::NoProc => ExitReason::NoProc, + LinkOutcome::Unchanged => panic!("the relink found a stale record"), + }; + assert_eq!(fresh[0].reason, expected); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// Three parties: `a` dies while `b` (linked) unlinks and `c` links to it. +/// `b` gets its notice only before its unlink returned; `c` gets exactly +/// one notice (the real reason or a late `NoProc`). +#[test] +fn death_racing_unlink_and_a_new_link() { + model("death_racing_unlink_and_a_new_link", || { + let (a, b, c) = ( + Arc::new(Actor::new(1)), + Arc::new(Actor::new(2)), + Arc::new(Actor::new(3)), + ); + assert_eq!(link(&b, &a, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[])) + }; + let linking = { + let (a, c) = (a.clone(), c.clone()); + spawn(move || link(&c, &a, 2, false)) + }; + unlink(&b, &a, 1); + dying.join().unwrap(); + let outcome = linking.join().unwrap(); + assert!(b.log().len() <= 1); + let expected = match outcome { + LinkOutcome::Linked => ExitReason::Panic, + LinkOutcome::NoProc => ExitReason::NoProc, + LinkOutcome::Unchanged => panic!("a first link is never unchanged"), + }; + assert_eq!( + c.log(), + vec![Notice { + from: a.id(), + gen: 2, + reason: expected + }] + ); + assert_eq!(a.records() + b.records() + c.records(), 0); + }); +} + +/// Both ends die at once: each notice takes effect at most once, and never +/// on a receiver past its own death point (its close took the record). +#[test] +fn mutual_death_delivers_at_most_one_notice() { + model("mutual_death_delivers_at_most_one_notice", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + assert_eq!(link(&a, &b, 1, false), LinkOutcome::Linked); + let dying = { + let a = a.clone(); + spawn(move || die(&a, ExitReason::Panic, &[1])) + }; + die(&b, ExitReason::Shutdown, &[1]); + dying.join().unwrap(); + assert!(a.log().len() + b.log().len() <= 1); + assert_eq!(a.records() + b.records(), 0); + }); +} + +/// Concurrent `link` from both ends: one edge, symmetric records. +#[test] +fn concurrent_links_install_one_edge() { + model("concurrent_links_install_one_edge", || { + let (a, b) = (Arc::new(Actor::new(1)), Arc::new(Actor::new(2))); + let other = { + let (a, b) = (a.clone(), b.clone()); + spawn(move || link(&b, &a, 2, false)) + }; + let mine = link(&a, &b, 1, false); + let theirs = other.join().unwrap(); + let outcomes = [mine, theirs]; + assert_eq!( + outcomes + .iter() + .filter(|o| **o == LinkOutcome::Linked) + .count(), + 1 + ); + assert!(outcomes.contains(&LinkOutcome::Unchanged)); + assert_eq!((a.records(), b.records()), (1, 1)); + // Release the records (no cycle is left behind). + die(&a, ExitReason::Panic, &[]); + die(&b, ExitReason::Panic, &[]); + }); +} + +// --------------------------------------------------------------------------- +// Monitors +// --------------------------------------------------------------------------- + +type Monitors = fence::Entries; + +/// A target: its exit table (D) and its publication flag. +struct Target { + actor: Actor, + published: AtomicBool, +} + +impl Target { + fn new() -> Arc { + Arc::new(Self { + actor: Actor::new(9), + published: AtomicBool::new(false), + }) + } + fn is_past_death(&self) -> bool { + table::lock(&self.actor.links).is_dead() + } + fn die(&self) { + drop(table::close( + self.actor.id(), + &self.actor.links, + &ExitReason::Panic, + |_| {}, + )); + self.published.store(true, Ordering::Release); + } +} + +/// The observer: its monitors, mailbox of `Down`s and fence flag. +struct Observer { + monitors: Monitors, + downs: loom::sync::Mutex>, + fenced: AtomicBool, + cancelled: AtomicBool, +} + +impl Observer { + fn new() -> Arc { + Arc::new(Self { + monitors: loom::sync::Mutex::new(Some(Default::default())), + downs: loom::sync::Mutex::new(Vec::new()), + fenced: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + }) + } + fn enqueue(&self, reason: ExitReason) { + assert!( + !self.fenced.load(Ordering::Acquire), + "a Down was enqueued after demonitor or the observer's close returned" + ); + let mut downs = self.downs.lock().unwrap(); + assert!(downs.is_empty(), "a second Down"); + downs.push(reason); + } + fn downs(&self) -> Vec { + self.downs.lock().unwrap().clone() + } + fn register(&self, target: &Target) -> fence::Registration { + fence::register( + &self.monitors, + 1, + (), + || false, + || target.is_past_death(), + || self.enqueue(ExitReason::NoProc), + || {}, + ) + } + /// The waiter: woken by publication or by its token. + fn waiter(self: &Arc, target: &Arc) -> loom::thread::JoinHandle { + let (observer, target) = (self.clone(), target.clone()); + spawn(move || loop { + if observer.cancelled.load(Ordering::Acquire) { + return false; + } + if target.published.load(Ordering::Acquire) { + return fence::deliver(&observer.monitors, &1, || { + observer.enqueue(ExitReason::Panic) + }); + } + loom::thread::yield_now(); + }) + } +} + +/// `demonitor` racing the target's death and the waiter: once it returns no +/// `Down` is enqueued; if it found an active entry none was, otherwise the +/// `Down` was delivered and is flushed (OTP `demonitor(Ref, [flush])`). +#[test] +fn demonitor_fences_the_down() { + model("demonitor_fences_the_down", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + let waiter = observer.waiter(&target); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + let removed = fence::demonitor(&observer.monitors, &1, || {}); + observer.fenced.store(true, Ordering::Release); + if removed.is_some() { + observer.cancelled.store(true, Ordering::Release); + } + dying.join().unwrap(); + let delivered = waiter.join().unwrap(); + assert_eq!(delivered, removed.is_none()); + let expected = if removed.is_some() { + vec![] + } else { + vec![ExitReason::Panic] + }; + assert_eq!(observer.downs(), expected); + assert!( + !fence::take_queued(&observer.monitors, &1), + "a queued Down survived demonitor" + ); + }); +} + +/// The observer's dispatch of a delivered `Down` racing `demonitor` on +/// another thread (a cloned context): a dispatch that begins after +/// `demonitor` returned never takes the `Down`; one that took it first +/// handles it (it was received before the flush). +#[test] +fn demonitor_flushes_a_down_in_dispatch() { + model("demonitor_flushes_a_down_in_dispatch", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + target.die(); + let waiter = observer.waiter(&target); + assert!(waiter.join().unwrap(), "the Down was delivered"); + let dispatcher = { + let observer = observer.clone(); + spawn(move || { + let fenced = observer.fenced.load(Ordering::Acquire); + let took = fence::take_queued(&observer.monitors, &1); + assert!(!(fenced && took), "a Down handled after demonitor returned"); + }) + }; + assert!(fence::demonitor(&observer.monitors, &1, || {}).is_none()); + observer.fenced.store(true, Ordering::Release); + dispatcher.join().unwrap(); + // Either the dispatch took it first, or the flush removed it. + assert!(!fence::take_queued(&observer.monitors, &1)); + }); +} + +/// `monitor` racing the target's death point: exactly one `Down`, the real +/// reason (registered before D) or `NoProc` (after D). +#[test] +fn monitor_racing_death_gets_exactly_one_down() { + model("monitor_racing_death_gets_exactly_one_down", || { + let (observer, target) = (Observer::new(), Target::new()); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + // Seen published before the call: the registration is after D. + let dead_before = target.published.load(Ordering::Acquire); + let registration = observer.register(&target); + if dead_before { + assert_eq!(registration, fence::Registration::Late, "monitor after D"); + } + let delivered = match registration { + fence::Registration::Active => observer.waiter(&target).join().unwrap(), + fence::Registration::Late => false, + fence::Registration::Inert => panic!("a running observer is never inert"), + }; + dying.join().unwrap(); + let expected = match registration { + fence::Registration::Active => { + assert!(delivered); + ExitReason::Panic + } + _ => ExitReason::NoProc, + }; + assert_eq!(observer.downs(), vec![expected]); + }); +} + +/// The observer's own exit racing the delivery: at most one `Down`, none +/// after the close returned; a monitor after the close is inert. +#[test] +fn observer_close_fences_the_down() { + model("observer_close_fences_the_down", || { + let (observer, target) = (Observer::new(), Target::new()); + assert_eq!(observer.register(&target), fence::Registration::Active); + let waiter = observer.waiter(&target); + let dying = { + let target = target.clone(); + spawn(move || target.die()) + }; + let taken = fence::close(&observer.monitors); + observer.fenced.store(true, Ordering::Release); + if taken.is_some_and(|entries| !entries.is_empty()) { + observer.cancelled.store(true, Ordering::Release); + } + assert_eq!(observer.register(&target), fence::Registration::Inert); + dying.join().unwrap(); + let delivered = waiter.join().unwrap(); + assert_eq!(observer.downs().len(), usize::from(delivered)); + }); +} diff --git a/concurrency/tests/mailbox_drop_panic.rs b/concurrency/tests/mailbox_drop_panic.rs index baa64cc4e..ad1e3349e 100644 --- a/concurrency/tests/mailbox_drop_panic.rs +++ b/concurrency/tests/mailbox_drop_panic.rs @@ -42,15 +42,27 @@ impl Drop for Victim { } impl Actor for Victim {} impl Handler for Victim { - async fn handle(&mut self, msg: Hold, _: &Context) { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { msg.0.send(()).unwrap(); let _ = msg.1.await; assert!(!msg.2, "original handler failure"); + reply.reply(()); } } impl Handler for Victim { - async fn handle(&mut self, _: Bomb, _: &Context) { + async fn handle( + &mut self, + _: Bomb, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } struct Ping; @@ -64,12 +76,25 @@ impl Actor for Observer { } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.send((false, down.reason)).unwrap(); + reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } // Not simulated: builds its own multi-thread Tokio runtime. @@ -126,7 +151,14 @@ fn mailbox_drop_child() { victim.send(Hold(entered, gate, handler_panics)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let replies: Vec<_> = (0..bombs) - .map(|_| victim.request_raw(Bomb(dropped.clone())).unwrap()) + .map(|_| { + victim + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); victim.context().stop(); release.send(()).unwrap(); @@ -139,11 +171,12 @@ fn mailbox_drop_child() { ExitReason::Panic("panic in handler: original handler failure".into()) ); } else { + // A destructor panic after the death point is a + // cleanup failure; the final reason stays Normal. + assert_eq!(reason, ExitReason::Normal); assert_eq!( - reason, - ExitReason::Panic( - "queued message destructor: queued message destructor".into() - ) + victim.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") ); } let first = tokio::time::timeout(LIMIT, events.recv()) @@ -216,29 +249,54 @@ fn abandoned_receiver_child() { let (entered, active) = oneshot::channel(); let (hold, gate) = oneshot::channel(); let owner = std::thread::spawn(move || { - let mut builder = if polled { tokio::runtime::Builder::new_multi_thread() } - else { tokio::runtime::Builder::new_current_thread() }; + let mut builder = if polled { + tokio::runtime::Builder::new_multi_thread() + } else { + tokio::runtime::Builder::new_current_thread() + }; let runtime = builder.worker_threads(1).enable_all().build().unwrap(); { let _entered = runtime.enter(); let actor = Victim(Arc::default(), state_panics).start(); - if polled { actor.send(Hold(entered, gate, false)).unwrap(); } + if polled { + actor.send(Hold(entered, gate, false)).unwrap(); + } send.send(actor).unwrap(); } stopped.recv_timeout(LIMIT).unwrap(); runtime.shutdown_timeout(LIMIT); }); let actor = tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); - if polled { tokio::time::timeout(LIMIT, active).await.unwrap().unwrap(); } + if polled { + tokio::time::timeout(LIMIT, active).await.unwrap().unwrap(); + } let dropped = Arc::new(AtomicUsize::new(0)); - let replies: Vec<_> = (0..2).map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()).collect(); + let replies: Vec<_> = (0..2) + .map(|_| { + actor + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) + .collect(); shutdown.send(()).unwrap(); - let reason = tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(); - if state_panics && !polled { assert!(matches!(reason, ExitReason::Panic(ref message) if message.contains("actor state destructor"))); } - else { assert_eq!(reason, ExitReason::Kill); } + let reason = tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(); + // A runtime drop ends Killed (G1); destructor panics are + // cleanup failures, never the reason. + let _ = state_panics; + assert_eq!(reason, ExitReason::Killed); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().mailbox_cleanup_failure().as_deref(), Some("queued message destructor")); - for reply in replies { assert!(tokio::time::timeout(LIMIT, reply).await.unwrap().is_err()); } + assert_eq!( + actor.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") + ); + for reply in replies { + assert!(tokio::time::timeout(LIMIT, reply).await.unwrap().is_err()); + } drop(hold); owner.join().unwrap(); } @@ -246,53 +304,95 @@ fn abandoned_receiver_child() { // A committed stop on a never-polled startup must also survive a // panicking actor-state destructor and two queued destructor failures. for committed in [ExitReason::Shutdown, ExitReason::Kill] { - let owner = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap(); + let owner = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); let actor = { let _entered = owner.enter(); Victim(Arc::default(), true).start() }; let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } actor.context().stop_with_reason(committed.clone()); owner.shutdown_background(); - assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), committed); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + committed + ); assert_eq!(dropped.load(Ordering::SeqCst), 2); } - for (managed, state_panics) in [(false, false), (false, true), (true, false), (true, true)] { - for committed in [ExitReason::Shutdown, ExitReason::Kill, ExitReason::Panic("linked peer cause".into())] { + for (managed, state_panics) in [(false, false), (false, true), (true, false), (true, true)] + { + for committed in [ + ExitReason::Shutdown, + ExitReason::Kill, + ExitReason::Panic("linked peer cause".into()), + ] { let system = ActorSystem::new().unwrap(); let group = system.async_group("preserve-cause", 1).unwrap(); - let actor = if managed { Victim(Arc::default(), state_panics).start_on(&group).unwrap() } - else { Victim(Arc::default(), state_panics).start() }; + let actor = if managed { + Victim(Arc::default(), state_panics) + .start_on(&group) + .unwrap() + } else { + Victim(Arc::default(), state_panics).start() + }; let (entered, arrival) = oneshot::channel(); let (release, gate) = oneshot::channel(); actor.send(Hold(entered, gate, false)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } if matches!(committed, ExitReason::Panic(_)) { let peer = Victim(Arc::default(), false).start(); actor.context().link(&peer.child_handle()); peer.context().stop_with_reason(committed.clone()); tokio::time::timeout(LIMIT, peer.wait_exit()).await.unwrap(); - } else { actor.context().stop_with_reason(committed.clone()); } + } else { + actor.context().stop_with_reason(committed.clone()); + } release.send(()).unwrap(); - assert_eq!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), committed); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + committed + ); assert_eq!(dropped.load(Ordering::SeqCst), 2); - assert_eq!(actor.child_handle().mailbox_cleanup_failure().as_deref(), Some("queued message destructor")); + assert_eq!( + actor.child_handle().cleanup_failure().as_deref(), + Some("queued message destructor") + ); assert!(system.shutdown(LIMIT).await.complete); } } // An actor-state panic must not unwind through an unguarded mailbox. + // After the death point it is a cleanup failure; the reason stays Normal. let actor = Victim(Arc::default(), true).start(); let (entered, arrival) = oneshot::channel(); let (release, gate) = oneshot::channel(); actor.send(Hold(entered, gate, false)).unwrap(); tokio::time::timeout(LIMIT, arrival).await.unwrap().unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); - for _ in 0..2 { actor.send(Bomb(dropped.clone())).unwrap(); } - actor.context().stop(); release.send(()).unwrap(); - assert!(matches!(tokio::time::timeout(LIMIT, actor.wait_exit()).await.unwrap(), ExitReason::Panic(_))); + for _ in 0..2 { + actor.send(Bomb(dropped.clone())).unwrap(); + } + actor.context().stop(); + release.send(()).unwrap(); + assert_eq!( + tokio::time::timeout(LIMIT, actor.wait_exit()) + .await + .unwrap(), + ExitReason::Normal + ); + assert!(actor.child_handle().cleanup_failure().is_some()); assert_eq!(dropped.load(Ordering::SeqCst), 2); }); runtime.shutdown_timeout(LIMIT); diff --git a/concurrency/tests/metrics.rs b/concurrency/tests/metrics.rs index e381fe06c..3b87e52a1 100644 --- a/concurrency/tests/metrics.rs +++ b/concurrency/tests/metrics.rs @@ -42,22 +42,45 @@ impl Actor for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: metrics_protocol::Query, _: &Context) -> u64 { - 9 + async fn handle( + &mut self, + _: metrics_protocol::Query, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(9); } } impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("metric panic regression"); } } impl Handler for Probe { - async fn handle(&mut self, hold: Hold, _: &Context) { + async fn handle( + &mut self, + hold: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = hold.entered.send(()); let _ = hold.release.await; + reply.reply(()); } } fn runtime() -> tokio::runtime::Runtime { @@ -71,14 +94,17 @@ fn runtime() -> tokio::runtime::Runtime { // reset, and these assertions must not race unrelated actors in other tests. #[test] fn metrics_conserve_work_across_all_lifecycle_paths() { - metrics::configure(metrics::Config { - timing_sample_every: 1, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(1)).unwrap(); runtime().block_on(async { let (release, gate) = oneshot::channel(); let actor = Probe { gate: Some(gate) }.start(); - let replies: Vec<_> = (0..10).map(|_| actor.request_raw(Ping).unwrap()).collect(); + let replies: Vec<_> = (0..10) + .map(|_| { + actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) + .collect(); assert_eq!(actor.mailbox_depth(), 10); assert_eq!(actor.context().mailbox_depth(), 10); assert!(matches!( @@ -86,11 +112,17 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { Err(ActorError::RequestTimeout) )); assert!(matches!( - Response::from_with_timeout(actor.request_raw(Ping), Duration::ZERO).await, + Response::from( + actor.send_request(Ping, spawned_concurrency::Deadline::after(Duration::ZERO)) + ) + .await, Err(ActorError::RequestTimeout) )); // Dropping either kind of caller wrapper does not cancel actor work. - drop(Response::from(actor.request_raw(Ping))); + drop(Response::from(actor.send_request( + Ping, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ))); let mut cancelled = Box::pin(actor.request(Ping)); assert!(futures::poll!(cancelled.as_mut()).is_pending()); drop(cancelled); @@ -107,28 +139,42 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { spawned_concurrency::tasks::request(recipient.as_ref(), Ping, Duration::from_secs(2)) .await .unwrap(); - Response::from(actor.request_raw(Ping)).await.unwrap(); - Response::from_with_timeout(actor.request_raw(Ping), Duration::from_secs(2)) + Response::from(actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT)) .await .unwrap(); + Response::from(actor.send_request( + Ping, + spawned_concurrency::Deadline::after(Duration::from_secs(2)), + )) + .await + .unwrap(); actor.context().stop(); assert_eq!(within!(actor.wait_exit()).await, ExitReason::Normal); assert_eq!(actor.mailbox_depth(), 0); assert!(actor.send(Ping).is_err()); assert!(matches!( - Response::from_with_timeout(actor.request_raw(Ping), Duration::from_secs(2)).await, - Err(ActorError::ActorStopped) + Response::from(actor.send_request( + Ping, + spawned_concurrency::Deadline::after(Duration::from_secs(2)) + )) + .await, + Err(ActorError::NoProc) )); let (release, gate) = oneshot::channel(); let actor = Probe { gate: Some(gate) }.start(); - let raw = actor.request_raw(Ping).unwrap(); + let raw = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); actor.send(Ping).unwrap(); let mut pending = Box::pin(actor.request(Ping)); assert!(futures::poll!(pending.as_mut()).is_pending()); actor.context().stop(); release.send(()).unwrap(); - assert!(matches!(pending.await, Err(ActorError::ActorStopped))); + assert!(matches!( + pending.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); assert!(raw.await.is_err()); within!(actor.join()).await; assert_eq!(actor.mailbox_depth(), 0); @@ -136,7 +182,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { let actor = Probe { gate: None }.start(); assert!(matches!( actor.request(Crash).await, - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); assert!(matches!( within!(actor.wait_exit()).await, @@ -190,7 +236,7 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { owner.block_on(entered).unwrap(); } drop(owner); - assert_eq!(actor.exit_reason(), Some(ExitReason::Kill)); + assert_eq!(actor.exit_reason(), Some(ExitReason::Killed)); assert_eq!(actor.mailbox_depth(), 0); } let rt = tokio::runtime::Builder::new_multi_thread() @@ -215,15 +261,32 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { struct Native; impl ThreadActor for Native {} impl ThreadHandler for Native { - fn handle(&mut self, _: metrics_protocol::Query, _: &ThreadContext) -> u64 { - 9 + fn handle( + &mut self, + _: metrics_protocol::Query, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &ThreadContext, + ) { + reply.reply(9); } } impl ThreadHandler for Native { - fn handle(&mut self, _: Ping, _: &ThreadContext) {} + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &ThreadContext, + ) { + reply.reply(()); + } } impl ThreadHandler for Native { - fn handle(&mut self, _: Crash, _: &ThreadContext) { + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &ThreadContext, + ) { panic!("native metric panic"); } } @@ -254,6 +317,8 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { + s.actors_exited_shutdown_total + s.actors_exited_panic_total + s.actors_exited_kill_total + + s.actors_exited_killed_total + + s.actors_exited_noproc_total ); assert_eq!( s.messages_enqueued_total, @@ -268,6 +333,8 @@ fn metrics_conserve_work_across_all_lifecycle_paths() { s.requests_replied_total + s.requests_timed_out_total + s.requests_actor_stopped_total + // Requests to an actor past its death point (rank 11 x 5e). + + s.requests_noproc_total + s.requests_cancelled_total ); assert!(s.actors_exited_shutdown_total >= 1); diff --git a/concurrency/tests/metrics_composable.rs b/concurrency/tests/metrics_composable.rs index c6f25309d..ee0e9b394 100644 --- a/concurrency/tests/metrics_composable.rs +++ b/concurrency/tests/metrics_composable.rs @@ -13,7 +13,14 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] @@ -22,25 +29,15 @@ fn embedded_components_can_add_sampling_requirements_after_actors_start() { actor.request(Ping).unwrap(); assert_eq!(metrics::snapshot().queue_wait.count, 1); // The implicit default did not register an application requirement. - metrics::configure_composable(Config { - timing_sample_every: 0, - }); + metrics::configure_composable(Config::default().timing_sample_every(0)); actor.request(Ping).unwrap(); assert_eq!(metrics::snapshot().queue_wait.count, 1); // Two libraries initialize independently, after the host is already running. - metrics::configure_composable(Config { - timing_sample_every: 8, - }); - metrics::configure_composable(Config { - timing_sample_every: 2, - }); - metrics::configure_composable(Config { - timing_sample_every: 0, - }); - metrics::configure_composable(Config { - timing_sample_every: 16, - }); + metrics::configure_composable(Config::default().timing_sample_every(8)); + metrics::configure_composable(Config::default().timing_sample_every(2)); + metrics::configure_composable(Config::default().timing_sample_every(0)); + metrics::configure_composable(Config::default().timing_sample_every(16)); assert_eq!(metrics::configuration().timing_sample_every, 2); for _ in 0..32 { actor.request(Ping).unwrap(); diff --git a/concurrency/tests/metrics_counts_only.rs b/concurrency/tests/metrics_counts_only.rs index 7108f5a7c..01c067962 100644 --- a/concurrency/tests/metrics_counts_only.rs +++ b/concurrency/tests/metrics_counts_only.rs @@ -11,14 +11,18 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn zero_timing_rate_retains_counters_without_timing_samples() { - metrics::configure(metrics::Config { - timing_sample_every: 0, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(0)).unwrap(); let actor = Probe.start(); for _ in 0..16 { actor.request(Ping).unwrap(); @@ -36,9 +40,7 @@ fn zero_timing_rate_retains_counters_without_timing_samples() { s.actors_active + s.handlers_active + s.requests_active + s.mailbox_depth, 0 ); - metrics::configure_composable(metrics::Config { - timing_sample_every: 8, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(8)); assert_eq!(metrics::configuration().timing_sample_every, 8); assert!(metrics::configure(metrics::Config::default()).is_err()); } diff --git a/concurrency/tests/metrics_deferred.rs b/concurrency/tests/metrics_deferred.rs new file mode 100644 index 000000000..f75fedfd9 --- /dev/null +++ b/concurrency/tests/metrics_deferred.rs @@ -0,0 +1,273 @@ +//! `spawned_deferred_*` and `spawned_jobs_*` metric families: every deferred +//! token and job outcome is counted once, and the gauges return to zero. +//! One test per binary: the counters are process-wide. +#![cfg(feature = "async")] +#![cfg(feature = "metrics")] +use spawned_concurrency::{ + message::Message, + metrics::{self, Config}, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobCtx, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }, + ActorError, ExitReason, +}; +use spawned_rt::tasks::{oneshot, sleep}; +use std::time::Duration; + +#[derive(Default)] +struct Owner { + stored: Vec>, + gates: Vec>, +} +impl Actor for Owner { + const MAX_OUTSTANDING: usize = 2; + const MAX_NATIVE_JOBS: Option = Some(1); +} + +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +enum Flush { + Reply, + Drop, +} +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Flush, reply: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + match msg { + Flush::Reply => { + token.reply(1); + } + Flush::Drop => drop(token), + } + } + reply.reply(()); + } +} + +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +struct Done; +impl Message for Done { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} +fn done(_: JobOutcome) -> Done { + Done +} + +enum Jobs { + Ok, + Panics, + Deadline, + Emits, + /// A native job finished, then cancelled: its queued result is fenced. + Fenced, + /// A gated native job holds the budget; a second one is rejected. + Budget, + Cancel, +} +impl Message for Jobs { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Jobs, reply: PendingReply<'_, ()>, ctx: &Context) { + match msg { + Jobs::Ok => drop(ctx.pipe(JobOptions::new(), |_| async { 1 }, done).unwrap()), + Jobs::Panics => drop( + ctx.pipe(JobOptions::new(), |_| async { panic!("job") }, done::<()>) + .unwrap(), + ), + Jobs::Deadline => { + let opts = JobOptions::new().deadline(Duration::from_millis(10)); + ctx.pipe(opts, |_| sleep(Duration::from_millis(60)), done) + .unwrap(); + } + Jobs::Emits => { + ctx.pipe( + JobOptions::new(), + |job: JobCtx| async move { + job.emit(Done).await.unwrap(); + job.emit(Done).await.unwrap(); + }, + done, + ) + .unwrap(); + } + Jobs::Fenced => { + let job = ctx + .pipe_blocking(JobOptions::new(), |_: JobCtx| 1, done) + .unwrap(); + while job.is_running() { + std::thread::sleep(Duration::from_millis(1)); + } + job.cancel(); + } + Jobs::Budget => { + let (open, gate) = std::sync::mpsc::channel::<()>(); + ctx.pipe_blocking( + JobOptions::new(), + move |_: JobCtx| gate.recv().ok(), + done, + ) + .unwrap(); + assert!(ctx + .pipe_blocking(JobOptions::new(), |_: JobCtx| (), done) + .is_err()); + open.send(()).unwrap(); + } + Jobs::Cancel => { + let (open, gate) = oneshot::channel::<()>(); + let job = ctx + .pipe( + JobOptions::new(), + |_| async move { gate.await.is_ok() }, + done, + ) + .unwrap(); + job.cancel(); + self.gates.push(open); + } + } + reply.reply(()); + } +} + +/// Wait until no job runs and their messages have been dispatched. +async fn settle(actor: &ActorRef) { + for _ in 0..400 { + if metrics::snapshot().jobs_running == 0 { + break; + } + sleep(Duration::from_millis(5)).await; + } + actor.request(Sync).await.unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn deferred_and_job_families_count_each_outcome_once() { + metrics::configure(Config::default().timing_sample_every(1)).unwrap(); + let before = metrics::snapshot(); + let actor = Owner::default().start(); + + // Replied, and Overloaded at the bound. + let first = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let second = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert!(matches!( + actor.request(Hold).await, + Err(ActorError::Overloaded) + )); + actor.request(Flush::Reply).await.unwrap(); + assert!(matches!(first.await, Ok(1))); + assert!(matches!(second.await, Ok(1))); + // Expired, dropped, abandoned. + let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(20)); + let expiring = actor.send_request(Hold, deadline).unwrap(); + assert!(matches!(expiring.await, Err(ActorError::RequestTimeout))); + let dropped = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Flush::Drop).await.unwrap(); + assert!(matches!(dropped.await, Err(ActorError::ReplyDropped))); + let abandoned = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + drop(abandoned); + actor.request(Flush::Reply).await.unwrap(); + + for jobs in [ + Jobs::Ok, + Jobs::Panics, + Jobs::Deadline, + Jobs::Emits, + Jobs::Fenced, + Jobs::Budget, + Jobs::Cancel, + ] { + actor.request(jobs).await.unwrap(); + // One job at a time: they share the actor's two credits. + settle(&actor).await; + } + + // Stopped: a token still stored at death. + let stopped = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Flush::Reply).await.unwrap(); + let late = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + sleep(Duration::from_millis(20)).await; + actor.context().stop(); + assert!(matches!( + late.await, + Err(ActorError::ActorStopped(ExitReason::Normal)) + )); + assert!(matches!(stopped.await, Ok(1))); + actor.join().await; + + let after = metrics::snapshot(); + let d = |f: fn(&metrics::Snapshot) -> u64| f(&after) - f(&before); + assert_eq!(d(|s| s.deferred_tokens_total), 7, "{after:#?}"); + assert_eq!(after.deferred_pending, before.deferred_pending); + assert_eq!(d(|s| s.deferred_replied_total), 3); + assert_eq!(d(|s| s.deferred_overloaded_total), 1); + assert_eq!(d(|s| s.deferred_expired_total), 1); + assert_eq!(d(|s| s.deferred_dropped_total), 1); + assert_eq!(d(|s| s.deferred_abandoned_total), 1); + assert_eq!(d(|s| s.deferred_stopped_total), 1); + assert_eq!( + after.deferred_reply_latency.count - before.deferred_reply_latency.count, + 3 + ); + assert_eq!(d(|s| s.jobs_started_total), 7); + assert_eq!(after.jobs_running, before.jobs_running); + // Ok, Emits, Budget's gated job, Fenced's (taken before the cancel). + assert_eq!(d(|s| s.jobs_completed_total), 4); + assert_eq!(d(|s| s.jobs_panicked_total), 1); + assert_eq!(d(|s| s.jobs_deadline_exceeded_total), 1); + assert_eq!(d(|s| s.jobs_emits_total), 2); + assert_eq!(d(|s| s.jobs_fenced_total), 1); + assert_eq!(d(|s| s.jobs_rejected_total), 1); + // Only Cancel's job was running when cancelled (Fenced's had finished; + // the owner's exit finds none running). + assert_eq!(d(|s| s.jobs_cancelled_total), 1); + let text = metrics::render_prometheus(); + for family in [ + "spawned_deferred_pending ", + "spawned_deferred_replied_total ", + "spawned_deferred_reply_latency_seconds_count ", + "spawned_jobs_running ", + "spawned_jobs_fenced_total ", + ] { + assert!(text.contains(family), "{family} missing"); + } +} diff --git a/concurrency/tests/metrics_sampling.rs b/concurrency/tests/metrics_sampling.rs index 5504acd07..28efcd77f 100644 --- a/concurrency/tests/metrics_sampling.rs +++ b/concurrency/tests/metrics_sampling.rs @@ -15,14 +15,18 @@ impl Message for Ping { } impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn sampling_retains_every_counter_and_scales_only_timing_observations() { - metrics::configure(metrics::Config { - timing_sample_every: 4, - }) - .unwrap(); + metrics::configure(metrics::Config::default().timing_sample_every(4)).unwrap(); let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() @@ -45,13 +49,9 @@ fn sampling_retains_every_counter_and_scales_only_timing_observations() { s.actors_active + s.handlers_active + s.requests_active + s.mailbox_depth, 0 ); - metrics::configure_composable(metrics::Config { - timing_sample_every: 8, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(8)); assert_eq!(metrics::configuration().timing_sample_every, 4); - metrics::configure_composable(metrics::Config { - timing_sample_every: 2, - }); + metrics::configure_composable(metrics::Config::default().timing_sample_every(2)); assert_eq!(metrics::configuration().timing_sample_every, 2); assert!(metrics::configure(metrics::Config::default()).is_err()); } diff --git a/concurrency/tests/native_dispatcher.rs b/concurrency/tests/native_dispatcher.rs index d88039726..e0d5f6ec8 100644 --- a/concurrency/tests/native_dispatcher.rs +++ b/concurrency/tests/native_dispatcher.rs @@ -16,9 +16,17 @@ impl Message for Ping { struct Healthy; impl Actor for Healthy {} impl Handler for Healthy { - async fn handle(&mut self, _: Ping, _: &Context) -> u64 { - tokio::task::yield_now().await; - 42 + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + tokio::task::yield_now().await; + 42 + }; + reply.reply(__reply_value); } } struct DropPanic; diff --git a/concurrency/tests/native_jobs.rs b/concurrency/tests/native_jobs.rs new file mode 100644 index 000000000..f8599c0ab --- /dev/null +++ b/concurrency/tests/native_jobs.rs @@ -0,0 +1,363 @@ +//! Tasks-mode blocking jobs: where their native body runs is reserved with +//! the native slot (so a failure is an error, never a lost job), and a +//! managed actor's job keeps its scope owned until the body ends. +#![cfg(all(feature = "async", not(spawned_sim)))] // native jobs are not simulated +use spawned_concurrency::{ + message::Message, + system::ActorSystem, + tasks::{Actor, Context, Handler, JobOptions, JobOutcome, PendingReply}, +}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, Arc, Mutex, +}; +use std::time::Duration; +#[cfg(target_os = "linux")] +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +struct Done; +impl Message for Done { + type Result = (); +} + +/// Starts one blocking job in `started()` that ignores cancellation and +/// runs until the test releases it. +struct Gated { + gate: Mutex>>, + running: Arc, +} +impl Actor for Gated { + async fn started(&mut self, ctx: &Context) { + let gate = self.gate.lock().unwrap().take().unwrap(); + let running = self.running.clone(); + ctx.pipe_blocking( + JobOptions::new(), + move |_| { + running.store(true, Ordering::SeqCst); + let _ = gate.recv(); + running.store(false, Ordering::SeqCst); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + } +} +impl Handler for Gated { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} +} + +/// Review P1 (Codex 7): a managed actor's blocking job holds a scope lease +/// until its body ends, so a system shutdown is not complete (and a +/// supervisor does not see the cleanup done) while the native work still +/// runs. The job used to run on the blocking pool without a lease: once the +/// actor's bounded job join expired, shutdown reported completion with the +/// job still executing, on an owned group and on a borrowed runtime alike. +async fn shutdown_waits_for_the_blocking_job(borrowed: bool) { + let system = ActorSystem::new().unwrap(); + let group = if borrowed { + system + .async_group_on("borrowed-jobs", &tokio::runtime::Handle::current()) + .unwrap() + } else { + system.async_group("owned-jobs", 1).unwrap() + }; + let (release, gate) = mpsc::channel(); + let running = Arc::new(AtomicBool::new(false)); + let actor = group + .start(Gated { + gate: Mutex::new(Some(gate)), + running: running.clone(), + }) + .unwrap(); + actor.ready().await.unwrap(); + for _ in 0..500 { + if running.load(Ordering::SeqCst) { + break; + } + tokio::time::sleep(Duration::from_millis(2)).await; + } + assert!(running.load(Ordering::SeqCst), "the job never started"); + let report = system.shutdown(Duration::from_millis(300)).await; + assert!(running.load(Ordering::SeqCst)); + assert!( + !report.complete, + "shutdown completed while a managed blocking job still ran" + ); + release.send(()).unwrap(); + let report = report.wait(Duration::from_secs(5)).await; + assert!( + report.complete, + "shutdown did not complete after the job ended" + ); + assert!(!running.load(Ordering::SeqCst)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_waits_for_a_managed_blocking_job() { + shutdown_waits_for_the_blocking_job(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn shutdown_on_a_borrowed_runtime_waits_for_a_managed_blocking_job() { + shutdown_waits_for_the_blocking_job(true).await; +} + +/// Review P1 (Codex 7): with a `shutdown_timeout` above the scope's default +/// job-join limit (5 s), the actor's root released its lease after that +/// limit while its blocking job still ran; the supervisor saw the cleanup +/// complete and started a replacement next to the old job's effects. The +/// job's lease keeps the old scope owned: no replacement until it ends. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_supervisor_does_not_replace_a_child_whose_blocking_job_still_runs() { + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use std::sync::atomic::AtomicUsize; + /// The first instance starts a gated job and exits (`Normal`; the + /// child is `Permanent`). + struct Child(Option); + impl Actor for Child { + async fn started(&mut self, ctx: &Context) { + if let Some(gated) = &mut self.0 { + let gate = gated.gate.lock().unwrap().take().unwrap(); + let running = gated.running.clone(); + ctx.pipe_blocking( + JobOptions::new(), + move |_| { + running.store(true, Ordering::SeqCst); + let _ = gate.recv(); + running.store(false, Ordering::SeqCst); + }, + |_: JobOutcome<()>| Done, + ) + .unwrap(); + // Exit after readiness (a startup exit is a startup failure). + ctx.send(Leave).unwrap(); + } + } + } + struct Leave; + impl Message for Leave { + type Result = (); + } + impl Handler for Child { + async fn handle(&mut self, _: Leave, _: PendingReply<'_, ()>, ctx: &Context) { + ctx.stop(); + } + } + impl Handler for Child { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let system = ActorSystem::new().unwrap(); + let group = system.async_group("supervised-jobs", 2).unwrap(); + let (release, gate) = mpsc::channel(); + let running = Arc::new(AtomicBool::new(false)); + let starts = Arc::new(AtomicUsize::new(0)); + let first = Arc::new(Mutex::new(Some(Gated { + gate: Mutex::new(Some(gate)), + running: running.clone(), + }))); + let mut builder = system.supervisor(SupervisorConfig::default()).unwrap(); + let factory_starts = starts.clone(); + builder + .add( + ChildSpec::new("gated", &group, move |_| { + factory_starts.fetch_add(1, Ordering::SeqCst); + let gated = first.lock().unwrap().take(); + async move { Ok::<_, String>(Child(gated)) } + }) + .shutdown_timeout(Duration::from_secs(10)), + ) + .unwrap(); + let _supervisor = builder.start().await.unwrap(); + // Past the scope's 5 s job-join limit, with the job still running. + tokio::time::sleep(Duration::from_millis(5600)).await; + assert!(running.load(Ordering::SeqCst), "the job ended early"); + assert_eq!( + starts.load(Ordering::SeqCst), + 1, + "a replacement started while the old blocking job still ran" + ); + release.send(()).unwrap(); + for _ in 0..1000 { + if starts.load(Ordering::SeqCst) == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert_eq!( + starts.load(Ordering::SeqCst), + 2, + "no replacement after the job ended" + ); + let _ = system.shutdown(Duration::from_secs(5)).await; +} + +/// Review P1 (Codex 6): job admission is one step with the owner's exit. +/// `pipe` builds the work (user code) between its exit check and +/// the admission; an owner that exits meanwhile refuses the job, so it +/// never runs. It used to be admitted into the job set the exit had already +/// drained: uncancelled work outliving its owner. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_job_admitted_after_its_owner_exited_never_runs() { + use spawned_concurrency::tasks::ActorStart; + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let actor = Owner.start(); + let ctx = actor.context(); + let owner = actor.clone(); + let ran = Arc::new(AtomicBool::new(false)); + let work_ran = ran.clone(); + let handle = tokio::task::spawn_blocking(move || { + ctx.pipe( + JobOptions::new(), + |job| { + // While the work is built, the owner exits and dies. + owner.context().stop(); + while owner.exit_reason().is_none() { + std::thread::sleep(Duration::from_millis(1)); + } + async move { + work_ran.store(true, Ordering::SeqCst); + job.cancelled().await; + } + }, + |_: JobOutcome<()>| Done, + ) + .unwrap() + }) + .await + .unwrap(); + assert!( + handle.is_cancelled(), + "a job admitted after its owner's exit escaped the exit's cancellation" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!ran.load(Ordering::SeqCst), "a refused job ran"); + actor.join().await; +} + +/// Review round 2 (Codex 4): an unused blocking permit held outside the +/// actor does not keep a dead generation's scope +/// owned. Its scope lease used to be taken at reservation, so the stopped +/// actor's scope could not join, and a system shutdown stayed incomplete, +/// until the external permit was dropped. The lease is now taken when the +/// job is admitted; an executing job still holds it until its body ends. +async fn an_unused_permit_does_not_hold_the_scope(borrowed: bool) { + use spawned_concurrency::tasks::{ActorStart, BlockingJobPermit}; + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + let system = ActorSystem::new().unwrap(); + let group = if borrowed { + system + .async_group_on("unused-permit-borrowed", &tokio::runtime::Handle::current()) + .unwrap() + } else { + system.async_group("unused-permit-owned", 1).unwrap() + }; + let actor = Owner.start_on(&group).unwrap(); + actor.ready().await.unwrap(); + // Reserved through a cloned context and kept outside the actor, unused. + let permit: BlockingJobPermit = actor + .context() + .reserve_blocking_job(JobOptions::new()) + .unwrap(); + actor.context().stop(); + assert_eq!( + actor.wait_exit().await, + spawned_concurrency::ExitReason::Normal + ); + let report = system.shutdown(Duration::from_secs(2)).await; + assert!( + report.complete, + "an unused blocking permit kept the dead actor's scope owned" + ); + // Using it now is refused: nothing runs. + let handle = permit.pipe_blocking(|_| (), |_| Done); + assert!(handle.is_cancelled()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unused_permit_does_not_hold_the_scope_owned_runtime() { + an_unused_permit_does_not_hold_the_scope(false).await; +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_unused_permit_does_not_hold_the_scope_borrowed_runtime() { + an_unused_permit_does_not_hold_the_scope(true).await; +} + +/// Review P1 (Codex 12, agy 6): outside a Tokio runtime a tasks actor's +/// blocking job runs on an owned thread. Its creation is part of the +/// reservation, so a failure is `SpawnFailed` (and `pipe_reply*` hands the +/// token back), as in threads mode. The thread used to be spawned after the +/// permit was granted and the token consumed: a failure was only logged, +/// the caller got a `JobHandle` and `done` never ran. +#[cfg(target_os = "linux")] +#[test] +fn a_failed_job_thread_spawn_is_a_reservation_error() { + subprocess::run( + "job_thread_spawn_failure_child", + "job thread spawn failure reported", + ); +} + +#[cfg(target_os = "linux")] +#[test] +#[ignore = "subprocess helper exercised by the enabled parent"] +fn job_thread_spawn_failure_child() { + use nix::sys::resource::{getrlimit, setrlimit, Resource}; + use spawned_concurrency::tasks::{ActorStart, Backend, PipeError}; + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + struct Owner; + impl Actor for Owner {} + impl Handler for Owner { + async fn handle(&mut self, _: Done, _: PendingReply<'_, ()>, _: &Context) {} + } + // The actor runs on its own thread; this thread has no Tokio runtime. + let actor = Owner.start_with_backend(Backend::Thread); + let ctx = actor.context(); + assert!(tokio::runtime::Handle::try_current().is_err()); + let (soft, hard) = getrlimit(Resource::RLIMIT_NPROC).unwrap(); + // No new threads for this user: the job thread cannot be created. + setrlimit(Resource::RLIMIT_NPROC, 0, hard).unwrap(); + if let Ok(probe) = std::thread::Builder::new().spawn(|| ()) { + // Not enforced here (a privileged user): nothing to observe. + let _ = probe.join(); + setrlimit(Resource::RLIMIT_NPROC, soft, hard).unwrap(); + ctx.stop(); + println!("job thread spawn failure reported (RLIMIT_NPROC not enforced)"); + return; + } + let reserved = ctx.reserve_blocking_job(JobOptions::new()); + let piped = ctx.pipe_blocking(JobOptions::new(), |_| (), |_| Done); + setrlimit(Resource::RLIMIT_NPROC, soft, hard).unwrap(); + match reserved { + Err(error) => assert!(matches!(error, PipeError::SpawnFailed(_)), "{error}"), + Ok(_) => panic!("the reservation succeeded without a job thread"), + } + assert!( + matches!(piped, Err(ref e) if matches!(e, PipeError::SpawnFailed(_))), + "pipe_blocking returned {piped:?}" + ); + assert_eq!( + ctx.deferred_outstanding(), + 0, + "a refused job kept its credit" + ); + // With threads available again, the same call runs the job. + let permit = ctx + .reserve_blocking_job(JobOptions::new()) + .expect("a job thread"); + drop(permit); + ctx.stop(); + std::thread::sleep(Duration::from_millis(50)); + println!("job thread spawn failure reported"); +} diff --git a/concurrency/tests/native_mailbox_drop.rs b/concurrency/tests/native_mailbox_drop.rs index 6b987c0bf..543bf6245 100644 --- a/concurrency/tests/native_mailbox_drop.rs +++ b/concurrency/tests/native_mailbox_drop.rs @@ -40,8 +40,15 @@ impl Drop for Victim { } } impl Handler for Victim { - fn handle(&mut self, _: Bomb, _: &Context) { - panic!("queued handler ran"); + fn handle( + &mut self, + _: Bomb, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("queued handler ran"); + } } } @@ -50,8 +57,15 @@ impl Message for Crash { type Result = (); } impl Handler for Victim { - fn handle(&mut self, _: Crash, _: &Context) { - panic!("native handler failure"); + fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("native handler failure"); + } } } @@ -83,7 +97,14 @@ fn native_drop_child() { arrival.recv_timeout(LIMIT).unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); let replies: Vec<_> = (0..2) - .map(|_| actor.request_raw(Bomb(dropped.clone())).unwrap()) + .map(|_| { + actor + .send_request( + Bomb(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); actor.context().stop_with_reason(reason.clone()); release.send(()).unwrap(); @@ -92,26 +113,17 @@ fn native_drop_child() { let (done, exit) = mpsc::channel(); std::thread::spawn(move || done.send(waiter.wait_exit_blocking()).unwrap()); let exit = exit.recv_timeout(LIMIT).unwrap(); - if reason != ExitReason::Normal { - assert_eq!(exit, reason); - } else if state_panics { - assert_eq!( - exit, - ExitReason::Panic("actor state destructor: native state destructor".into()) - ); - } else { - assert_eq!( - exit, - ExitReason::Panic("queued message destructor: native queued destructor".into()) - ); - } + // Destructor panics after the death point are cleanup failures: + // the final reason is the requested one. + let _ = state_panics; + assert_eq!(exit, reason); assert_eq!( - child.mailbox_cleanup_failure().as_deref(), + child.cleanup_failure().as_deref(), Some("native queued destructor") ); assert_eq!(dropped.load(Ordering::SeqCst), 2); for reply in replies { - assert!(reply.recv_timeout(LIMIT).is_err()); + assert!(reply.recv().is_err()); } // Keep ActorRef alive through all assertions: cleanup cannot depend // on whoever eventually drops its last sender. diff --git a/concurrency/tests/nested_supervision.rs b/concurrency/tests/nested_supervision.rs index f3180c86e..d05677509 100644 --- a/concurrency/tests/nested_supervision.rs +++ b/concurrency/tests/nested_supervision.rs @@ -25,16 +25,12 @@ use std::{ mod tree_record; const LIMIT: Duration = Duration::from_secs(5); fn policy(retries: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts: retries, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(retries).backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false), + ) } struct Next; impl Message for Next { @@ -47,14 +43,29 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.0 += 1; - self.0 + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { + self.0 += 1; + self.0 + }; + reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("nested leaf failed"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("nested leaf failed"); + } } } type Published = Arc>>>; @@ -212,7 +223,9 @@ async fn failed_nested_startup_is_cleaned_before_parent_retries() { "fallible", &placement, move |_| async move { - if attempt == 0 { + // v0.8 (S1): the parent's first start of the nested + // tree is not retried, so its replacement fails. + if attempt == 1 { Err("nested initial construction failed".to_string()) } else { Ok(Counter(0)) @@ -227,8 +240,13 @@ async fn failed_nested_startup_is_cleaned_before_parent_retries() { .await .unwrap() .unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); - assert_eq!(nested.snapshot().generation, 2); + nested.current().unwrap().stop(); + spawned_rt::tasks::timeout(LIMIT, nested.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(nested.snapshot().generation, 3); assert!(nested.current().is_ok()); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); @@ -413,6 +431,11 @@ async fn native_leaf_helper_blocks_subtree_replacement_and_remains_owned_after_a struct HeldStop(Arc); impl Actor for HeldStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.0.notified().await; } @@ -578,8 +601,15 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("nested leaf failed"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("nested leaf failed"); + } } } diff --git a/concurrency/tests/otp_exit_signals.rs b/concurrency/tests/otp_exit_signals.rs new file mode 100644 index 000000000..4f9a16a38 --- /dev/null +++ b/concurrency/tests/otp_exit_signals.rs @@ -0,0 +1,1561 @@ +//! Exit signals, kills, NoProc and parent exits as in OTP +//! (docs/design/otp-exit-signals.md, ranks 5e and 5g). Each test names the +//! design rule it pins; every one fails on v0.7 (the API or the outcome +//! differs there). +#![cfg(feature = "async")] +// Real Tokio runtimes and OS threads (plain `#[tokio::test]`, threads mode): +// not run under the simulator. `exit_signals_sim.rs` explores the same +// orderings under `spawned_sim`. +#![cfg(not(spawned_sim))] +use spawned_concurrency::{message::Message, ActorId, Down, Exit, ExitReason, NoProc}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::Duration, +}; + +const LIMIT: Duration = Duration::from_secs(10); + +/// What an actor observed: `exit_received` calls, `Down`s and whether +/// `stopped()` ran. +#[derive(Default, Clone)] +struct Seen { + exits: Arc>>, + downs: Arc>>, + stopped: Arc, +} +impl Seen { + fn exits(&self) -> Vec { + self.exits.lock().unwrap().clone() + } + fn downs(&self) -> Vec { + self.downs.lock().unwrap().clone() + } + fn stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } +} + +struct Ping; +impl Message for Ping { + type Result = u8; +} + +#[path = "support/drop_subprocess.rs"] +mod subprocess; + +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, ActorRef, ActorStart, Context, Handler}; + use spawned_concurrency::ChildHandle; + + pub struct Node { + pub seen: Seen, + pub trap: bool, + /// stopped() waits for this signal (to act while it runs). + pub in_stopped: Option<( + tokio::sync::oneshot::Sender<()>, + tokio::sync::oneshot::Receiver<()>, + )>, + pub panic_in_stopped: bool, + /// Deferred tokens stored by `Hold`. + pub held: Vec>, + /// stopped() drops the stored tokens before waiting on `in_stopped`. + pub drop_held_in_stopped: bool, + } + impl Node { + pub fn new(seen: &Seen, trap: bool) -> Self { + Self { + seen: seen.clone(), + trap, + in_stopped: None, + panic_in_stopped: false, + held: Vec::new(), + drop_held_in_stopped: false, + } + } + } + impl Actor for Node { + async fn started(&mut self, ctx: &Context) { + if self.trap { + ctx.trap_exit(true); + } + } + async fn stopped(&mut self, _: &Context) { + self.seen.stopped.store(true, Ordering::SeqCst); + if self.drop_held_in_stopped { + self.held.clear(); + } + if let Some((entered, release)) = self.in_stopped.take() { + let _ = entered.send(()); + let _ = release.await; + } + if self.panic_in_stopped { + panic!("stopped() panic after the kill"); + } + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.seen.exits.lock().unwrap().push(exit); + } + } + impl Handler for Node { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + impl Handler for Node { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + self.seen.downs.lock().unwrap().push(down); + reply.reply(()); + } + } + type Action = Box) + Send>; + struct Do(Action); + impl Message for Do { + type Result = (); + } + impl Handler for Node { + async fn handle( + &mut self, + f: Do, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + (f.0)(ctx); + reply.reply(()); + } + } + /// Defer the request and store its token. + struct Hold; + impl Message for Hold { + type Result = u8; + } + impl Handler for Node { + async fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + self.held.push(reply.defer().unwrap()); + } + } + /// Panic in a request handler (the request's obligation unwinds). + struct Boom; + impl Message for Boom { + type Result = u8; + } + impl Handler for Node { + async fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + panic!("boom in a request handler"); + } + } + /// Block the actor in a handler until released. + struct Block(tokio::sync::oneshot::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Node { + async fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let _ = msg.0.await; + reply.reply(()); + } + } + /// Run `f` inside one of `actor`'s handlers. + pub async fn on(actor: &ActorRef, f: impl FnOnce(&Context) + Send + 'static) { + actor.request(Do(Box::new(f))).await.unwrap(); + } + pub async fn ended(child: &ChildHandle) -> ExitReason { + tokio::time::timeout(LIMIT, child.wait_exit_async()) + .await + .expect("actor did not exit") + } + /// A request round trip, so everything queued before it was handled. + pub async fn settle(actor: &ActorRef) { + assert_eq!(actor.request(Ping).await.unwrap(), 7); + } + + // E1: kill() is untrappable and ends the target Killed without stopped(); + // links receive Killed (trappable), monitors Down { Killed }. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn kill_is_untrappable_and_propagates_killed() { + let (target_seen, peer_seen, watcher_seen, trapper_seen) = ( + Seen::default(), + Seen::default(), + Seen::default(), + Seen::default(), + ); + let target = Node::new(&target_seen, true).start(); + let peer = Node::new(&peer_seen, false).start(); + let trapper = Node::new(&trapper_seen, true).start(); + let watcher = Node::new(&watcher_seen, false).start(); + settle(&target).await; + settle(&trapper).await; + let t = target.child_handle(); + let t2 = t.clone(); + on(&peer, move |ctx| ctx.link(&t2)).await; + let t3 = t.clone(); + on(&trapper, move |ctx| ctx.link(&t3)).await; + let t4 = t.clone(); + on(&watcher, move |ctx| { + ctx.monitor(&t4); + }) + .await; + t.kill(); + assert_eq!(ended(&t).await, ExitReason::Killed); + assert!(!target_seen.stopped(), "stopped() runs after a kill"); + assert!(target_seen.exits().is_empty(), "a kill was trapped"); + // A non-trapping peer ends Killed (and skips stopped()). + assert_eq!(ended(&peer.child_handle()).await, ExitReason::Killed); + assert!(!peer_seen.stopped()); + // A trapping peer gets the notice as a message. + settle(&trapper).await; + assert_eq!( + trapper_seen.exits(), + vec![Exit::new(t.id(), ExitReason::Killed)] + ); + // The Down is queued by the monitor's waiter after publication + // (GUIDE: "shortly after"), so wait for it rather than one round + // trip, which a single CPU can finish first. + let began = std::time::Instant::now(); + while watcher_seen.downs().is_empty() { + assert!(began.elapsed() < LIMIT, "no Down after the kill"); + settle(&watcher).await; + tokio::time::sleep(Duration::from_millis(2)).await; + } + assert_eq!(watcher_seen.downs().len(), 1); + assert_eq!(watcher_seen.downs()[0].reason, ExitReason::Killed); + trapper.child_handle().stop(); + watcher.child_handle().stop(); + ended(&trapper.child_handle()).await; + ended(&watcher.child_handle()).await; + } + + // E1 + precedence: a kill during stopped() replaces the requested reason; + // the running stopped() finishes and a panic in it does not replace Killed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn kill_during_stopped_supersedes_and_panic_does_not_replace_it() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + node.panic_in_stopped = true; + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + child.kill(); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Killed); + assert!(seen.stopped(), "a stopped() that began finishes"); + } + + // E1/decision 5: an administrative stop_with_reason(Kill) stays an + // administrative stop ending Kill, and stopped() runs. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn administrative_kill_reason_runs_stopped_and_ends_kill() { + let seen = Seen::default(); + let actor = Node::new(&seen, true).start(); + settle(&actor).await; + actor.child_handle().stop_with_reason(ExitReason::Kill); + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Kill); + assert!(seen.stopped()); + } + + // E2: exit(Normal) is ignored by a non-trapping actor and trapped as an + // Exit from EXTERNAL by a trapping one; any other reason ends a + // non-trapping actor with it, without stopped(). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn exit_signals_follow_exit_signal_2() { + let plain_seen = Seen::default(); + let plain = Node::new(&plain_seen, false).start(); + settle(&plain).await; + plain.child_handle().exit(ExitReason::Normal); + settle(&plain).await; + assert!( + plain.child_handle().is_alive(), + "Normal stopped a non-trapping actor" + ); + + let trap_seen = Seen::default(); + let trapper = Node::new(&trap_seen, true).start(); + settle(&trapper).await; + trapper.child_handle().exit(ExitReason::Normal); + trapper.child_handle().exit(ExitReason::Shutdown); + settle(&trapper).await; + assert_eq!( + trap_seen.exits(), + vec![ + Exit::new(ActorId::EXTERNAL, ExitReason::Normal), + Exit::new(ActorId::EXTERNAL, ExitReason::Shutdown), + ] + ); + assert!(trapper.child_handle().is_alive()); + + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()).await, ExitReason::Shutdown); + assert!( + !plain_seen.stopped(), + "stopped() ran after an untrapped signal" + ); + trapper.child_handle().stop(); + ended(&trapper.child_handle()).await; + } + + // E2: ctx.exit sends from the calling actor; a Kill from an actor is a kill. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn ctx_exit_sends_from_the_calling_actor() { + let (a_seen, b_seen) = (Seen::default(), Seen::default()); + let a = Node::new(&a_seen, false).start(); + let b = Node::new(&b_seen, true).start(); + settle(&b).await; + let bh = b.child_handle(); + let bh2 = bh.clone(); + on(&a, move |ctx| { + ctx.exit(&bh2, ExitReason::Panic("from a".into())) + }) + .await; + settle(&b).await; + assert_eq!( + b_seen.exits(), + vec![Exit::new(a.id(), ExitReason::Panic("from a".into()))] + ); + let bh3 = bh.clone(); + on(&a, move |ctx| ctx.exit(&bh3, ExitReason::Kill)).await; + assert_eq!(ended(&bh).await, ExitReason::Killed); + a.child_handle().stop(); + ended(&a.child_handle()).await; + } + + // E4 + decision 1: link/monitor to an actor past its death point give + // NoProc; a non-trapping caller exits NoProc without stopped(); + // try_link returns Err(NoProc) and keeps nothing. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn registrations_after_death_report_noproc() { + let dead_seen = Seen::default(); + let dead = Node::new(&dead_seen, false).start(); + dead.child_handle() + .stop_with_reason(ExitReason::Panic("gone".into())); + assert!(matches!( + ended(&dead.child_handle()).await, + ExitReason::Panic(_) + )); + let d = dead.child_handle(); + + let trap_seen = Seen::default(); + let trapper = Node::new(&trap_seen, true).start(); + let (d1, d2) = (d.clone(), d.clone()); + on(&trapper, move |ctx| { + ctx.link(&d1); + ctx.monitor(&d2); + }) + .await; + settle(&trapper).await; + assert_eq!( + trap_seen.exits(), + vec![Exit::new(d.id(), ExitReason::NoProc)] + ); + let downs = trap_seen.downs(); + assert_eq!(downs.len(), 1); + assert_eq!(downs[0].reason, ExitReason::NoProc); + + let try_seen = Seen::default(); + let trier = Node::new(&try_seen, false).start(); + let result = Arc::new(Mutex::new(None)); + let (d3, r) = (d.clone(), result.clone()); + on(&trier, move |ctx| { + *r.lock().unwrap() = Some(ctx.try_link(&d3)) + }) + .await; + assert_eq!(*result.lock().unwrap(), Some(Err(NoProc))); + settle(&trier).await; + assert!(trier.child_handle().is_alive()); + + let plain_seen = Seen::default(); + let plain = Node::new(&plain_seen, false).start(); + let d4 = d.clone(); + on(&plain, move |ctx| ctx.link(&d4)).await; + assert_eq!(ended(&plain.child_handle()).await, ExitReason::NoProc); + assert!(!plain_seen.stopped()); + + trapper.child_handle().stop(); + trier.child_handle().stop(); + ended(&trapper.child_handle()).await; + ended(&trier.child_handle()).await; + } + + // Sender lifetime / semantics X2: a Context kept by a helper after its + // actor exited cannot link: no record, no notice, no NoProc. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn link_from_a_dead_actor_is_inert() { + let (dead_seen, peer_seen) = (Seen::default(), Seen::default()); + let dead = Node::new(&dead_seen, true).start(); + let peer = Node::new(&peer_seen, true).start(); + settle(&peer).await; + let kept = dead.context(); + dead.child_handle().stop(); + ended(&dead.child_handle()).await; + kept.link(&peer.child_handle()); + assert_eq!(kept.try_link(&peer.child_handle()), Ok(())); + // The live peer gets no edge: its own exit notifies nobody. + let gone = Node::new(&Seen::default(), false).start(); + kept.link(&gone.child_handle()); + gone.child_handle().stop(); + ended(&gone.child_handle()).await; + peer.child_handle().exit(ExitReason::Panic("peer".into())); + settle(&peer).await; + peer.child_handle() + .stop_with_reason(ExitReason::Panic("peer".into())); + assert!(matches!( + ended(&peer.child_handle()).await, + ExitReason::Panic(_) + )); + assert!(dead_seen.exits().is_empty()); + assert_eq!(dead.child_handle().exit_reason(), Some(ExitReason::Normal)); + } + + // E3 with decision 2: supervisor shutdown is the parent's exit signal. + // A non-trapping child ends Shutdown without stopped(); a trapping child + // runs stopped(); a child that does not reach its death point within + // shutdown_timeout is killed. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn supervisor_shutdown_is_the_parents_exit_signal() { + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + let system = ActorSystem::new().unwrap(); + let group = system.async_group("e3", 1).unwrap(); + let (plain_seen, trap_seen, stuck_seen) = + (Seen::default(), Seen::default(), Seen::default()); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (hold, hold_rx) = tokio::sync::oneshot::channel::<()>(); + let stuck_args = Arc::new(Mutex::new(Some((entered_tx, hold_rx)))); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let (p, t, k) = (plain_seen.clone(), trap_seen.clone(), stuck_seen.clone()); + let plain = tree + .add(ChildSpec::new("plain", &group, move |_| { + let seen = p.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let trapping = tree + .add(ChildSpec::new("trapping", &group, move |_| { + let seen = t.clone(); + async move { Ok::<_, String>(Node::new(&seen, true)) } + })) + .unwrap(); + let stuck = tree + .add( + ChildSpec::new("stuck", &group, move |_| { + let seen = k.clone(); + let gate = stuck_args.lock().unwrap().take().unwrap(); + async move { + let mut node = Node::new(&seen, true); + node.in_stopped = Some(gate); + Ok::<_, String>(node) + } + }) + .shutdown_timeout(Duration::from_millis(50)), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let (plain, trapping, stuck) = ( + plain.current().unwrap().child_handle(), + trapping.current().unwrap().child_handle(), + stuck.current().unwrap().child_handle(), + ); + let shutdown = tokio::spawn(async move { supervisor.shutdown(LIMIT).await.complete }); + entered.await.unwrap(); + // The stuck stopped() outlives shutdown_timeout (50 ms): the kill + // lands while it runs; Rust cannot interrupt it, so release it after. + tokio::time::sleep(Duration::from_millis(300)).await; + drop(hold); + assert!(shutdown.await.unwrap()); + assert_eq!(ended(&plain).await, ExitReason::Shutdown); + assert!(!plain_seen.stopped(), "a non-trapping child ran stopped()"); + assert_eq!(ended(&trapping).await, ExitReason::Shutdown); + assert!(trap_seen.stopped()); + assert!( + trap_seen.exits().is_empty(), + "the parent exit reached exit_received" + ); + assert_eq!(ended(&stuck).await, ExitReason::Killed); + assert!(stuck_seen.stopped()); + assert!(system.shutdown(LIMIT).await.complete); + } + + pub struct Parent { + pub child: Option>, + pub child_seen: Seen, + } + impl Actor for Parent {} + struct Spawn; + impl Message for Spawn { + type Result = ChildHandle; + } + impl Handler for Parent { + async fn handle( + &mut self, + _: Spawn, + reply: spawned_concurrency::PendingReply<'_, ChildHandle>, + ctx: &Context, + ) { + let child = Node::new(&self.child_seen, true).start_linked(ctx); + reply.reply(child.child_handle()); + self.child = Some(child); + } + } + + // E3: a start_linked parent's exit, Normal included, ends a trapping + // child through stopped(); exit_received never sees it. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn parent_exit_ends_a_trapping_child_normally() { + for parent_reason in [ExitReason::Normal, ExitReason::Shutdown] { + let child_seen = Seen::default(); + let parent = Parent { + child: None, + child_seen: child_seen.clone(), + } + .start(); + let child = parent.request(Spawn).await.unwrap(); + parent + .child_handle() + .stop_with_reason(parent_reason.clone()); + assert_eq!(ended(&child).await, parent_reason); + assert!(child_seen.stopped(), "the parent exit skipped stopped()"); + assert!( + child_seen.exits().is_empty(), + "exit_received saw the parent exit" + ); + } + } + + // Phase 0: signals that arrive while started() runs are resolved when it + // returns, with the trap flag read then. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn signals_during_started_are_resolved_with_the_flag_set_in_started() { + struct Slow(Seen, Option>); + impl Actor for Slow { + async fn started(&mut self, ctx: &Context) { + self.1.take().unwrap().await.unwrap(); + ctx.trap_exit(true); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.0.exits.lock().unwrap().push(exit); + } + } + impl Handler for Slow { + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + let seen = Seen::default(); + let (go, gate) = tokio::sync::oneshot::channel(); + let actor = Slow(seen.clone(), Some(gate)).start(); + actor.child_handle().exit(ExitReason::Shutdown); + go.send(()).unwrap(); + assert_eq!(actor.request(Ping).await.unwrap(), 7); + assert_eq!( + seen.exits(), + vec![Exit::new(ActorId::EXTERNAL, ExitReason::Shutdown)] + ); + actor.child_handle().stop(); + ended(&actor.child_handle()).await; + } + + // Decision 8: a late non-kill signal replaces the requested reason of a + // non-trapping actor before D; a running stopped() finishes. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn late_signal_replaces_the_requested_reason() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop(); + entered.await.unwrap(); + child.exit(ExitReason::Shutdown); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + assert!(seen.stopped()); + } + + // Context::exit_pending: a trapping supervised child sees the + // supervisor's queued parent exit in the handlers of messages queued + // before it; a trapped Exit from a peer is not a pending exit. + mod exit_pending { + use super::*; + use spawned_concurrency::supervision::{ChildSpec, SupervisorConfig}; + use spawned_concurrency::system::ActorSystem; + + type Log = Arc>>; + pub struct Late(Log); + impl Actor for Late { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + self.0.lock().unwrap().push(("started", ctx.exit_pending())); + } + async fn exit_received(&mut self, _: Exit, ctx: &Context) { + self.0.lock().unwrap().push(("exit", ctx.exit_pending())); + } + async fn stopped(&mut self, ctx: &Context) { + self.0.lock().unwrap().push(("stopped", ctx.exit_pending())); + } + } + struct Probe; + impl Message for Probe { + type Result = (); + } + impl Handler for Late { + async fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.0.lock().unwrap().push(("probe", ctx.exit_pending())); + reply.reply(()); + } + } + /// Wait (bounded) until a shutdown is pending, then record it. + struct AwaitShutdown; + impl Message for AwaitShutdown { + type Result = (); + } + impl Handler for Late { + async fn handle( + &mut self, + _: AwaitShutdown, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let deadline = tokio::time::Instant::now() + Duration::from_secs(3); + while !ctx.exit_pending() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(5)).await; + } + self.0.lock().unwrap().push(("await", ctx.exit_pending())); + reply.reply(()); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_work_sees_the_supervisors_pending_shutdown() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("pending", 1).unwrap(); + let log: Log = Arc::default(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let l = log.clone(); + let child = tree + .add( + ChildSpec::new("late", &group, move |_| { + let log = l.clone(); + async move { Ok::<_, String>(Late(log)) } + }) + // No kill escalation while the test waits. + .shutdown_timeout(LIMIT), + ) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let child = child.current().unwrap(); + // Normal operation, and a trapped Exit from outside: not pending. + child.request(Probe).await.unwrap(); + child.child_handle().exit(ExitReason::Shutdown); + child.request(Probe).await.unwrap(); + // Work queued before the supervisor's shutdown. + child.send(AwaitShutdown).unwrap(); + child.send(Probe).unwrap(); + child.send(Probe).unwrap(); + let handle = child.child_handle(); + assert!(supervisor.shutdown(LIMIT).await.complete); + assert_eq!(ended(&handle).await, ExitReason::Shutdown); + assert_eq!( + *log.lock().unwrap(), + vec![ + ("started", false), + ("probe", false), + ("exit", false), + ("probe", false), + ("await", true), + ("probe", true), + ("probe", true), + ("stopped", true), + ] + ); + assert!(system.shutdown(LIMIT).await.complete); + } + + // A non-trapping actor: an untrapped signal begins exit, so the + // handler that is running when it arrives sees it pending; ctx.stop() + // is pending at once. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn begun_exits_are_pending() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + let pending = Arc::new(Mutex::new(Vec::new())); + let p = pending.clone(); + on(&actor, move |ctx| { + p.lock().unwrap().push(ctx.exit_pending()); + ctx.stop(); + p.lock().unwrap().push(ctx.exit_pending()); + }) + .await; + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Normal); + assert_eq!(*pending.lock().unwrap(), vec![false, true]); + let actor = Node::new(&seen, false).start(); + let handle = actor.child_handle(); + let p = pending.clone(); + on(&actor, move |ctx| { + handle.exit(ExitReason::Shutdown); + p.lock().unwrap().push(ctx.exit_pending()); + }) + .await; + assert_eq!(ended(&actor.child_handle()).await, ExitReason::Shutdown); + assert_eq!(*pending.lock().unwrap(), vec![false, true, true]); + } + } + + // Decision 2 is easy to miss: a skipped stopped() is logged at debug + // (`event = "stopped_skipped"`), for a kill, an untrapped signal and a + // runtime drop. + mod stopped_skipped_log { + use super::*; + use crate::subprocess; + use std::collections::BTreeMap; + type Fields = BTreeMap; + #[derive(Clone, Default)] + struct Capture(Arc>>); + impl tracing::Subscriber for Capture { + fn enabled(&self, _: &tracing::Metadata<'_>) -> bool { + true + } + fn new_span(&self, _: &tracing::span::Attributes<'_>) -> tracing::span::Id { + tracing::span::Id::from_u64(1) + } + fn record(&self, _: &tracing::span::Id, _: &tracing::span::Record<'_>) {} + fn record_follows_from(&self, _: &tracing::span::Id, _: &tracing::span::Id) {} + fn enter(&self, _: &tracing::span::Id) {} + fn exit(&self, _: &tracing::span::Id) {} + fn event(&self, event: &tracing::Event<'_>) { + struct Visitor(Fields); + impl tracing::field::Visit for Visitor { + fn record_debug( + &mut self, + field: &tracing::field::Field, + value: &dyn std::fmt::Debug, + ) { + self.0.insert(field.name().to_owned(), format!("{value:?}")); + } + fn record_str(&mut self, field: &tracing::field::Field, value: &str) { + self.0.insert(field.name().to_owned(), value.to_owned()); + } + } + let mut visitor = Visitor(Fields::new()); + event.record(&mut visitor); + self.0 + .lock() + .unwrap() + .push((*event.metadata().level(), visitor.0)); + } + } + impl Capture { + /// (cause, reason) of each `stopped_skipped` event for `actor`. + fn skipped(&self, actor: ActorId) -> Vec<(String, String)> { + self.0 + .lock() + .unwrap() + .iter() + .filter(|(level, f)| { + *level == tracing::Level::DEBUG + && f.get("event").map(String::as_str) == Some("stopped_skipped") + && f.get("actor") == Some(&actor.to_string()) + }) + .map(|(_, f)| (f["cause"].clone(), f["reason"].clone())) + .collect() + } + } + + /// Runs in its own process with `Capture` as the global subscriber: + /// a thread-scoped subscriber raced other test threads registering + /// the `stopped_skipped` callsite, which could cache its interest as + /// "never" and drop the events (a flaky failure under load). + #[test] + fn skipped_stopped_is_logged_at_debug() { + subprocess::run( + "tasks_mode::stopped_skipped_log::child", + "skipped stopped() logged", + ); + } + + #[test] + #[ignore = "subprocess helper exercised by the enabled parent"] + fn child() { + if std::env::var_os("SPAWNED_MAILBOX_DROP_CHILD").is_none() { + return; + } + let capture = Capture::default(); + tracing::subscriber::set_global_default(capture.clone()).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let (killed, signalled, trapping) = runtime.block_on(async { + let seen = Seen::default(); + let killed = Node::new(&seen, true).start(); + settle(&killed).await; + killed.child_handle().kill(); + assert_eq!(ended(&killed.child_handle()).await, ExitReason::Killed); + let signalled = Node::new(&seen, false).start(); + settle(&signalled).await; + signalled.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&signalled.child_handle()).await, ExitReason::Shutdown); + assert!(!seen.stopped()); + // stopped() runs: nothing logged. + let trapping = Node::new(&seen, true).start(); + settle(&trapping).await; + trapping.child_handle().stop(); + assert_eq!(ended(&trapping.child_handle()).await, ExitReason::Normal); + assert!(seen.stopped()); + (killed.id(), signalled.id(), trapping.id()) + }); + assert_eq!( + capture.skipped(killed), + vec![("kill".into(), "killed".into())] + ); + assert_eq!( + capture.skipped(signalled), + vec![("exit_signal".into(), "shutdown".into())] + ); + assert!(capture.skipped(trapping).is_empty()); + // Runtime drop before stopped() began. + let dropped = runtime.block_on(async { + let actor = Node::new(&Seen::default(), true).start(); + settle(&actor).await; + actor.id() + }); + drop(runtime); + assert_eq!( + capture.skipped(dropped), + vec![("runtime_drop".into(), "killed".into())] + ); + println!("skipped stopped() logged"); + } + } + + // Rank 11 x 5e ("Required changes to rank 11"): callers, `Down` and + // `wait_exit` see one reason. Each test pins one rule of the design. + mod caller_errors { + use super::*; + use spawned_concurrency::tasks::ActorRef; + use spawned_concurrency::ActorError; + + async fn answered(rx: spawned_concurrency::ReplyReceiver) -> Result + where + R: Send + 'static, + { + tokio::time::timeout(LIMIT, rx) + .await + .expect("the request was never answered") + } + + /// Block `actor` in a handler; returns the release. + async fn block(actor: &ActorRef) -> tokio::sync::oneshot::Sender<()> { + let (release, gate) = tokio::sync::oneshot::channel(); + actor.send(Block(gate)).unwrap(); + release + } + + fn stopped_with(result: Result, reason: &ExitReason) { + match result { + Err(ActorError::ActorStopped(got)) => assert_eq!(&got, reason), + other => panic!("expected ActorStopped({reason:?}), got {other:?}"), + } + } + + // A request queued, or a token stored, at D is answered with the + // final reason: Killed after a kill (stopped() skipped). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_and_stored_requests_get_killed_after_a_kill() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + let stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + settle(&actor).await; + let release = block(&actor).await; + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let child = actor.child_handle(); + child.kill(); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Killed); + stopped_with(answered(queued).await, &ExitReason::Killed); + stopped_with(answered(stored).await, &ExitReason::Killed); + } + + // An untrapped exit signal: queued requests get the signal's reason. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn queued_request_gets_the_untrapped_signals_reason() { + let seen = Seen::default(); + let actor = Node::new(&seen, false).start(); + settle(&actor).await; + let release = block(&actor).await; + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let child = actor.child_handle(); + let reason = ExitReason::Panic("signalled".into()); + child.exit(reason.clone()); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, reason); + stopped_with(answered(queued).await, &reason); + } + + // A request whose handler panicked is not answered at once: it waits + // for D (stopped() runs first) and gets the final reason. A kill + // during stopped() makes that reason Killed for the caller too. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn panicked_request_is_answered_at_death_with_the_final_reason() { + for kill in [false, true] { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let mut crashed = actor + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + entered.await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + crashed.try_recv().is_none(), + "the panicked request was answered before D" + ); + let child = actor.child_handle(); + if kill { + child.kill(); + } + release.send(()).unwrap(); + let reason = ended(&child).await; + if kill { + assert_eq!(reason, ExitReason::Killed); + } else { + assert!(matches!(reason, ExitReason::Panic(_)), "{reason:?}"); + } + stopped_with(answered(crashed).await, &reason); + } + } + + // A token dropped while the owner exits is held until D, then + // answered with the final reason. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn token_dropped_while_exiting_is_held_until_death() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, false); + node.in_stopped = Some((entered_tx, release_rx)); + node.drop_held_in_stopped = true; + let actor = node.start(); + let mut stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + stored.try_recv().is_none(), + "a token dropped in stopped() was answered before D" + ); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + stopped_with(answered(stored).await, &ExitReason::Shutdown); + } + + // A request sent while the actor exits (a slow stopped() of a + // trapping actor) is admitted and answered at D; after D, send and + // request report NoProc. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn admitted_while_exiting_noproc_after_death() { + let seen = Seen::default(); + let (entered_tx, entered) = tokio::sync::oneshot::channel(); + let (release, release_rx) = tokio::sync::oneshot::channel(); + let mut node = Node::new(&seen, true); + node.in_stopped = Some((entered_tx, release_rx)); + let actor = node.start(); + settle(&actor).await; + let child = actor.child_handle(); + child.stop_with_reason(ExitReason::Shutdown); + entered.await.unwrap(); + let admitted = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("admitted while exiting"); + actor.send(Ping).expect("a send while exiting is admitted"); + release.send(()).unwrap(); + assert_eq!(ended(&child).await, ExitReason::Shutdown); + stopped_with(answered(admitted).await, &ExitReason::Shutdown); + assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); + assert!(matches!(actor.request(Ping).await, Err(ActorError::NoProc))); + assert!(matches!( + actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT), + Err(ActorError::NoProc) + )); + let recipient = actor.recipient::(); + assert!(matches!(recipient.send(Ping), Err(ActorError::NoProc))); + } + } + + mod startup { + use super::*; + use spawned_concurrency::supervision::{ + ChildSpec, ChildState, RestartMode, SupervisorConfig, SupervisorError, + SupervisorStrategy, + }; + use spawned_concurrency::system::ActorSystem; + use std::sync::atomic::AtomicUsize; + + fn terminal(state: ChildState) -> bool { + matches!(state, ChildState::Stopped | ChildState::Failed) + } + + // S1: a child's first start that fails aborts start() whatever the + // restart type: no retry (v0.7 retried Permanent and Transient + // children and timeouts within the restart budget). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn first_start_failure_aborts_start_whatever_the_restart_type() { + for (mode, timeout) in [ + (RestartMode::Permanent, false), + (RestartMode::Transient, false), + (RestartMode::Permanent, true), + (RestartMode::Transient, true), + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s1", 1).unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let seen = Seen::default(); + tree.add(ChildSpec::new("first", &group, move |_| { + let seen = seen.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let counted = calls.clone(); + tree.add( + ChildSpec::new("failing", &group, move |_| { + counted.fetch_add(1, Ordering::SeqCst); + async move { + if timeout { + std::future::pending::<()>().await; + } + Err::("boot failed") + } + }) + .restart(mode) + .startup_timeout(Duration::from_millis(50)), + ) + .unwrap(); + let result = tokio::time::timeout(LIMIT, tree.start()).await.unwrap(); + match result { + Err(SupervisorError::FailedToStartChild { child, .. }) => { + assert_eq!(child, "failing") + } + other => panic!("{mode:?}/{timeout}: unexpected {:?}", other.err()), + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "{mode:?}: first start retried" + ); + assert!(system.shutdown(LIMIT).await.complete); + } + } + + // S1: a ready child's exit during the initial start is recorded and + // processed after the tree commits ready, under the strategy (v0.7: + // OneForOne restarted at once, OneForAll/RestForOne failed start()). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn ready_child_exit_during_startup_is_processed_afterwards() { + for strategy in [ + SupervisorStrategy::OneForOne, + SupervisorStrategy::OneForAll, + SupervisorStrategy::RestForOne, + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s1", 1).unwrap(); + let config = SupervisorConfig::default().strategy(strategy); + let mut tree = system.supervisor(config).unwrap(); + let seen = Seen::default(); + let first = tree + .add(ChildSpec::new("first", &group, move |_| { + let seen = seen.clone(); + async move { Ok::<_, String>(Node::new(&seen, false)) } + })) + .unwrap(); + let (entered_tx, entered) = tokio::sync::oneshot::channel::<()>(); + let (release, release_rx) = tokio::sync::oneshot::channel::<()>(); + let gate = Arc::new(Mutex::new(Some((entered_tx, release_rx)))); + let second = tree + .add(ChildSpec::new("second", &group, move |_| { + let gate = gate.lock().unwrap().take(); + let seen = Seen::default(); + async move { + if let Some((entered, release)) = gate { + let _ = entered.send(()); + let _ = release.await; + } + Ok::<_, String>(Node::new(&seen, false)) + } + })) + .unwrap(); + let start = tokio::spawn(tree.start()); + entered.await.unwrap(); + let handle = first.current().unwrap().child_handle(); + handle.kill(); + assert_eq!(ended(&handle).await, ExitReason::Killed); + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + first.snapshot().generation, + 1, + "{strategy:?}: restarted before the initial start completed" + ); + release.send(()).unwrap(); + let supervisor = tokio::time::timeout(LIMIT, start) + .await + .unwrap() + .unwrap() + .unwrap_or_else(|e| panic!("{strategy:?}: start failed: {e}")); + tokio::time::timeout(LIMIT, first.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + if strategy == SupervisorStrategy::OneForOne { + assert_eq!(second.snapshot().generation, 1); + } else { + tokio::time::timeout(LIMIT, second.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + } + assert!(supervisor.shutdown(LIMIT).await.complete); + assert!(system.shutdown(LIMIT).await.complete); + } + } + + // S2: start() returns its error only after the rollback finished: + // every started child is terminal, including one whose stopped() + // gets stuck (killed at shutdown_timeout, then stuck after a second + // interval), whose cleanup failure is listed separately. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_returns_its_error_after_rollback() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("s2", 2).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let slow_seen = Seen::default(); + let s = slow_seen.clone(); + let slow = tree + .add(ChildSpec::new("slow", &group, move |_| { + let seen = s.clone(); + let (tx, rx) = tokio::sync::oneshot::channel(); + // stopped() waits until the sender's timer fires. + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + let _ = tx.send(()); + }); + let (entered, _) = tokio::sync::oneshot::channel(); + async move { + let mut node = Node::new(&seen, true); + node.in_stopped = Some((entered, rx)); + Ok::<_, String>(node) + } + })) + .unwrap(); + let (_hold, hold_rx) = tokio::sync::oneshot::channel::<()>(); + let hold_rx = Arc::new(Mutex::new(Some(hold_rx))); + let stuck = tree + .add( + ChildSpec::new("stuck", &group, move |_| { + let rx = hold_rx.lock().unwrap().take().unwrap(); + let (entered, _) = tokio::sync::oneshot::channel(); + async move { + let mut node = Node::new(&Seen::default(), true); + node.in_stopped = Some((entered, rx)); + Ok::<_, String>(node) + } + }) + .shutdown_timeout(Duration::from_millis(50)), + ) + .unwrap(); + tree.add( + ChildSpec::new("failing", &group, |_| async { + Err::("boot failed") + }) + .restart(RestartMode::Temporary), + ) + .unwrap(); + let result = tokio::time::timeout(LIMIT, tree.start()).await.unwrap(); + assert!(slow_seen.stopped(), "start() returned before rollback"); + assert!(terminal(slow.snapshot().state), "{:?}", slow.snapshot()); + assert_eq!(stuck.snapshot().state, ChildState::Failed); + match result { + Err(SupervisorError::FailedToStartChild { + child, + rollback_failures, + .. + }) => { + assert_eq!(child, "failing"); + assert!( + rollback_failures.iter().any(|f| f.contains("stuck")), + "{rollback_failures:?}" + ); + } + other => panic!("unexpected {:?}", other.err()), + } + let _ = system.shutdown(Duration::from_millis(100)).await; + } + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, ActorStart, Context, Handler}; + use spawned_concurrency::ChildHandle; + + struct Node(Seen, bool); + impl Actor for Node { + fn started(&mut self, ctx: &Context) { + if self.1 { + ctx.trap_exit(true); + } + } + fn stopped(&mut self, _: &Context) { + self.0.stopped.store(true, Ordering::SeqCst); + } + fn exit_received(&mut self, exit: Exit, _: &Context) { + self.0.exits.lock().unwrap().push(exit); + } + } + impl Handler for Node { + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, u8>, + _: &Context, + ) { + reply.reply(7); + } + } + fn ended(child: &ChildHandle) -> ExitReason { + let (tx, rx) = std::sync::mpsc::channel(); + let c = child.clone(); + std::thread::spawn(move || tx.send(c.wait_exit_blocking()).unwrap()); + rx.recv_timeout(LIMIT).expect("actor did not exit") + } + + #[test] + fn kill_exit_and_noproc_in_threads_mode() { + // Kill: untrappable, Killed, no stopped(); the trapping peer gets Killed. + let (target_seen, peer_seen) = (Seen::default(), Seen::default()); + let target = Node(target_seen.clone(), true).start(); + let peer = Node(peer_seen.clone(), true).start(); + assert_eq!(target.request(Ping).unwrap(), 7); + assert_eq!(peer.request(Ping).unwrap(), 7); + peer.context().link(&target.child_handle()); + target.child_handle().kill(); + assert_eq!(ended(&target.child_handle()), ExitReason::Killed); + assert!(!target_seen.stopped()); + assert_eq!(peer.request(Ping).unwrap(), 7); + assert_eq!( + peer_seen.exits(), + vec![Exit::new(target.id(), ExitReason::Killed)] + ); + // NoProc for a late link and try_link. + assert_eq!(peer.context().try_link(&target.child_handle()), Err(NoProc)); + peer.context().link(&target.child_handle()); + assert_eq!(peer.request(Ping).unwrap(), 7); + assert_eq!(peer_seen.exits().last().unwrap().reason, ExitReason::NoProc); + // exit(Normal) ignored by a non-trapping actor; exit(Shutdown) ends it + // without stopped(). + let plain_seen = Seen::default(); + let plain = Node(plain_seen.clone(), false).start(); + plain.child_handle().exit(ExitReason::Normal); + assert_eq!(plain.request(Ping).unwrap(), 7); + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()), ExitReason::Shutdown); + assert!(!plain_seen.stopped()); + peer.child_handle().stop(); + assert_eq!(ended(&peer.child_handle()), ExitReason::Normal); + } + + // Rank 11 x 5e in threads mode: the final reason at D, held exit errors, + // admission while exiting and NoProc after D. + mod caller_errors { + use super::*; + use spawned_concurrency::ActorError; + use spawned_concurrency::{PendingReply, ReplyReceiver, ReplyTo}; + use std::sync::mpsc; + + #[derive(Default)] + struct Server { + held: Vec>, + /// stopped() drops the held tokens, reports entry, then waits. + in_stopped: Option<(mpsc::Sender<()>, mpsc::Receiver<()>)>, + } + impl Actor for Server { + fn stopped(&mut self, _: &Context) { + self.held.clear(); + if let Some((entered, release)) = self.in_stopped.take() { + entered.send(()).unwrap(); + let _ = release.recv(); + } + } + } + impl Handler for Server { + fn handle(&mut self, _: Ping, reply: PendingReply<'_, u8>, _: &Context) { + reply.reply(7); + } + } + struct Hold; + impl Message for Hold { + type Result = u8; + } + impl Handler for Server { + fn handle(&mut self, _: Hold, reply: PendingReply<'_, u8>, _: &Context) { + self.held.push(reply.defer().unwrap()); + } + } + struct Boom; + impl Message for Boom { + type Result = u8; + } + impl Handler for Server { + fn handle(&mut self, _: Boom, _reply: PendingReply<'_, u8>, _: &Context) { + panic!("boom in a request handler"); + } + } + struct Block(mpsc::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Server { + fn handle(&mut self, msg: Block, reply: PendingReply<'_, ()>, _: &Context) { + let _ = msg.0.recv(); + reply.reply(()); + } + } + + fn stopped_with(rx: ReplyReceiver, reason: &ExitReason) { + match rx.recv() { + Err(ActorError::ActorStopped(got)) => assert_eq!(&got, reason), + other => panic!("expected ActorStopped({reason:?}), got {other:?}"), + } + } + + fn gated() -> (Server, mpsc::Receiver<()>, mpsc::Sender<()>) { + let (entered_tx, entered) = mpsc::channel(); + let (release, release_rx) = mpsc::channel(); + let server = Server { + in_stopped: Some((entered_tx, release_rx)), + ..Server::default() + }; + (server, entered, release) + } + + #[test] + fn queued_and_stored_requests_get_killed_after_a_kill() { + let actor = Server::default().start(); + let stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert_eq!(actor.request(Ping).unwrap(), 7); + let (release, gate) = mpsc::channel(); + actor.send(Block(gate)).unwrap(); + let queued = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.child_handle().kill(); + release.send(()).unwrap(); + assert_eq!(ended(&actor.child_handle()), ExitReason::Killed); + stopped_with(queued, &ExitReason::Killed); + stopped_with(stored, &ExitReason::Killed); + } + + #[test] + fn panicked_request_and_dropped_token_wait_for_death() { + let (server, entered, release) = gated(); + let actor = server.start(); + let mut stored = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let mut crashed = actor + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + entered.recv_timeout(LIMIT).unwrap(); + std::thread::sleep(Duration::from_millis(50)); + assert!( + crashed.try_recv().is_none(), + "panicked request answered before D" + ); + assert!( + stored.try_recv().is_none(), + "dropped token answered before D" + ); + release.send(()).unwrap(); + let reason = ended(&actor.child_handle()); + assert!(matches!(reason, ExitReason::Panic(_)), "{reason:?}"); + stopped_with(crashed, &reason); + stopped_with(stored, &reason); + } + + #[test] + fn admitted_while_exiting_noproc_after_death() { + let (server, entered, release) = gated(); + let actor = server.start(); + assert_eq!(actor.request(Ping).unwrap(), 7); + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + entered.recv_timeout(LIMIT).unwrap(); + let admitted = actor + .send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .expect("admitted while exiting"); + actor.send(Ping).expect("a send while exiting is admitted"); + release.send(()).unwrap(); + assert_eq!(ended(&actor.child_handle()), ExitReason::Shutdown); + stopped_with(admitted, &ExitReason::Shutdown); + assert!(matches!(actor.send(Ping), Err(ActorError::NoProc))); + assert!(matches!(actor.request(Ping), Err(ActorError::NoProc))); + assert!(matches!( + actor.send_request(Ping, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT), + Err(ActorError::NoProc) + )); + } + } + + // Context::exit_pending in threads mode: the start_linked parent's exit + // to a trapping child is pending in the handlers of messages queued + // before it. + mod exit_pending { + use super::*; + use spawned_concurrency::threads::ActorRef; + use std::sync::mpsc; + + type Log = Arc>>; + pub struct Late(Log); + impl Actor for Late { + fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + fn stopped(&mut self, ctx: &Context) { + self.0.lock().unwrap().push(("stopped", ctx.exit_pending())); + } + } + struct Probe; + impl Message for Probe { + type Result = (); + } + impl Handler for Late { + fn handle( + &mut self, + _: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + self.0.lock().unwrap().push(("probe", ctx.exit_pending())); + reply.reply(()); + } + } + struct Block(mpsc::Receiver<()>); + impl Message for Block { + type Result = (); + } + impl Handler for Late { + fn handle( + &mut self, + msg: Block, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let _ = msg.0.recv(); + self.0.lock().unwrap().push(("block", ctx.exit_pending())); + reply.reply(()); + } + } + struct Parent(Log); + impl Actor for Parent {} + struct Spawn; + impl Message for Spawn { + type Result = ActorRef; + } + impl Handler for Parent { + fn handle( + &mut self, + _: Spawn, + reply: spawned_concurrency::PendingReply<'_, ActorRef>, + ctx: &Context, + ) { + reply.reply(Late(self.0.clone()).start_linked(ctx)); + } + } + + #[test] + fn queued_work_sees_the_parents_pending_exit() { + let log: Log = Arc::default(); + let parent = Parent(log.clone()).start(); + let child = parent.request(Spawn).unwrap(); + child.request(Probe).unwrap(); + let (release, gate) = mpsc::channel(); + child.send(Block(gate)).unwrap(); + child.send(Probe).unwrap(); + child.send(Probe).unwrap(); + parent.child_handle().stop_with_reason(ExitReason::Shutdown); + // Link notices are delivered before the parent's exit is + // published: the child's parent exit is queued now. + assert_eq!(ended(&parent.child_handle()), ExitReason::Shutdown); + release.send(()).unwrap(); + assert_eq!(ended(&child.child_handle()), ExitReason::Shutdown); + assert_eq!( + *log.lock().unwrap(), + vec![ + ("probe", false), + ("block", true), + ("probe", true), + ("probe", true), + ("stopped", true), + ] + ); + } + } +} diff --git a/concurrency/tests/panic_safety.rs b/concurrency/tests/panic_safety.rs index 1b4fa5193..123783b93 100644 --- a/concurrency/tests/panic_safety.rs +++ b/concurrency/tests/panic_safety.rs @@ -61,7 +61,9 @@ fn expected(fault: Fault) -> ExitReason { Fault::DownHandler => panic("panic in handler: injected Down panic"), Fault::MessageDrop => panic("panic in handler: injected message drop panic"), Fault::ReplyDrop => panic("panic in handler: injected reply drop panic"), - Fault::QueuedMessageDrop => panic("queued message destructor: injected queued drop panic"), + // A destructor panic after the death point is a cleanup failure + // (`ChildHandle::cleanup_failure`), never the exit reason. + Fault::QueuedMessageDrop => ExitReason::Normal, } } @@ -195,41 +197,85 @@ mod tasks_actors { } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Poke, _: &Context) { + async fn handle( + &mut self, + _: Poke, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fault(Fault::SendHandler, "injected handler panic"); + reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Ask, _: &Context) -> u64 { - self.0.ask() + async fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0.ask()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Payload, _: &Context) {} + async fn handle( + &mut self, + _: Payload, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Gift, _: &Context) -> Bomb { - self.0.gift() + async fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::PendingReply<'_, Bomb>, + _: &Context, + ) { + reply.reply(self.0.gift()); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Hold, _: &Context) { + async fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.gate.wait(); + reply.reply(()); } } impl Handler for TaskVictim { - async fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), + async fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::PendingReply<'_, Option>, + ctx: &Context, + ) { + let __reply_value: Option = async { + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None } - None + .await; + reply.reply(__reply_value); } } impl Handler for TaskVictim { - async fn handle(&mut self, _: Down, _: &Context) { + async fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fault(Fault::DownHandler, "injected Down panic"); + reply.reply(()); } } @@ -243,25 +289,45 @@ mod tasks_actors { } } impl Handler for TaskObserver { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); + reply.reply(()); } } impl Handler for TaskObserver { - async fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), + async fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::PendingReply<'_, Option>, + ctx: &Context, + ) { + let __reply_value: Option = async { + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None } - None + .await; + reply.reply(__reply_value); } } impl Handler for TaskObserver { - async fn handle(&mut self, _: Ask, _: &Context) -> u64 { - 7 + async fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(7); } } } @@ -286,41 +352,84 @@ mod thread_actors { } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Poke, _: &Context) { + fn handle( + &mut self, + _: Poke, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fault(Fault::SendHandler, "injected handler panic"); + reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Ask, _: &Context) -> u64 { - self.0.ask() + fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0.ask()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Payload, _: &Context) {} + fn handle( + &mut self, + _: Payload, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Gift, _: &Context) -> Bomb { - self.0.gift() + fn handle( + &mut self, + _: Gift, + reply: spawned_concurrency::PendingReply<'_, Bomb>, + _: &Context, + ) { + reply.reply(self.0.gift()); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Hold, _: &Context) { + fn handle( + &mut self, + _: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.gate.wait(); + reply.reply(()); } } impl Handler for ThreadVictim { - fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), - } - None + fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::PendingReply<'_, Option>, + ctx: &Context, + ) { + let __reply_value: Option = (|| { + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None + })(); + reply.reply(__reply_value); } } impl Handler for ThreadVictim { - fn handle(&mut self, _: Down, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fault(Fault::DownHandler, "injected Down panic"); + reply.reply(()); } } @@ -334,25 +443,44 @@ mod thread_actors { } } impl Handler for ThreadObserver { - fn handle(&mut self, down: Down, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0 .events .push(Event::Down(down.monitor_ref, down.reason)); + reply.reply(()); } } impl Handler for ThreadObserver { - fn handle(&mut self, setup: Setup, ctx: &Context) -> Option { - match setup { - Setup::Trap => ctx.trap_exit(true), - Setup::Link(target) => ctx.link(&target), - Setup::Watch(target) => return Some(ctx.monitor(&target)), - } - None + fn handle( + &mut self, + setup: Setup, + reply: spawned_concurrency::PendingReply<'_, Option>, + ctx: &Context, + ) { + let __reply_value: Option = (|| { + match setup { + Setup::Trap => ctx.trap_exit(true), + Setup::Link(target) => ctx.link(&target), + Setup::Watch(target) => return Some(ctx.monitor(&target)), + } + None + })(); + reply.reply(__reply_value); } } impl Handler for ThreadObserver { - fn handle(&mut self, _: Ask, _: &Context) -> u64 { - 7 + fn handle( + &mut self, + _: Ask, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(7); } } } @@ -390,7 +518,7 @@ fn exercise(h: &dyn Harness, fault: Fault, gate: &Gate, events: &Events) -> Exit let trigger = h.spawn_trigger(); h.victim_setup(Setup::Trap); h.victim_setup(Setup::Link(trigger.clone())); - trigger.shutdown(); + trigger.stop_with_reason(spawned_concurrency::ExitReason::Shutdown); } Fault::DownHandler => { let trigger = h.spawn_trigger(); @@ -422,20 +550,31 @@ fn exercise(h: &dyn Harness, fault: Fault, gate: &Gate, events: &Events) -> Exit ); assert_eq!(h.observer_ask(), 7, "{fault:?}: observer stopped working"); let seen = events.snapshot(); + // A victim that panics in started() can pass its death point before the + // observer's monitor or link is registered: that registration gets NoProc. + let carries = + |r: &ExitReason| *r == reason || (fault == Fault::Started && *r == ExitReason::NoProc); assert_eq!( seen.iter() - .filter(|e| matches!(e, Event::Down(m, r) if *m == monitor && *r == reason)) + .filter(|e| matches!(e, Event::Down(m, r) if *m == monitor && carries(r))) .count(), 1, "{fault:?}: {seen:?}" ); assert_eq!( seen.iter() - .filter(|e| matches!(e, Event::Exit(id, r) if *id == victim.id() && *r == reason)) + .filter(|e| matches!(e, Event::Exit(id, r) if *id == victim.id() && carries(r))) .count(), 1, "{fault:?}: {seen:?}" ); + if fault == Fault::QueuedMessageDrop { + assert_eq!( + victim.cleanup_failure().as_deref(), + Some("injected queued drop panic"), + "{fault:?}" + ); + } assert_eq!(seen.len(), 2, "{fault:?}: extra notification {seen:?}"); reason } @@ -475,7 +614,11 @@ mod threads_mode { self.victim.request_with_timeout(Ask, LIMIT).map_err(|_| ()) } fn victim_abandoned_gift(&self) { - drop(self.victim.request_raw(Gift).unwrap()); + drop( + self.victim + .send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } fn victim_setup(&self, setup: Setup) -> Option { self.victim.request_with_timeout(setup, LIMIT).unwrap() @@ -649,7 +792,11 @@ mod tasks_mode { .map_err(|_| ()) } fn victim_abandoned_gift(&self) { - drop(self.victim.request_raw(Gift).unwrap()); + drop( + self.victim + .send_request(Gift, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } fn victim_setup(&self, setup: Setup) -> Option { self.env @@ -738,17 +885,13 @@ mod tasks_mode { } fn config() -> SupervisorConfig { - SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ) } /// Under supervision the first generation fails in each callback; the @@ -770,17 +913,21 @@ mod tasks_mode { let child_gate = gate.clone(); let child = tree .add(ChildSpec::new("victim", &group, move |ctx| { - let fault = (ctx.generation() == 1).then_some(fault); + // v0.8 (S1): a first start is not retried, so a + // started() panic is injected into the replacement. + let faulty = if fault == Fault::Started { 2 } else { 1 }; + let fault = (ctx.generation() == faulty).then_some(fault); let victim = TaskVictim(Victim::new(fault, &child_gate)); async move { Ok::<_, String>(victim) } })) .unwrap(); let supervisor = env.bounded(tree.start()).unwrap(); if fault == Fault::Started { - // The first generation never became ready; the retry did. - let next = env.bounded(child.wait_ready()).unwrap(); + // The replacement never became ready; its retry did. + child.current().unwrap().child_handle().kill(); + let next = env.bounded(child.wait_for_generation(2)).unwrap(); assert_eq!(env.bounded(next.request(Ask)).unwrap(), 1); - assert_eq!(child.snapshot().generation, 2); + assert_eq!(child.snapshot().generation, 3); assert_eq!(child.snapshot().last_exit, Some(expected(fault))); } else { let first = child.current().unwrap(); @@ -825,7 +972,9 @@ mod tasks_mode { let mut tree = env.system.as_ref().unwrap().supervisor(config()).unwrap(); let child = tree .add(ChildSpec::new("factory", &group, move |ctx| { - let first = ctx.generation() == 1; + // v0.8 (S1): a first start is not retried, so the + // panic is injected into the replacement. + let first = ctx.generation() == 2; if first && !in_future { panic!("injected factory panic"); } @@ -839,10 +988,11 @@ mod tasks_mode { })) .unwrap(); let supervisor = env.bounded(tree.start()).unwrap(); - let actor = child.current().unwrap(); + child.current().unwrap().child_handle().kill(); + let actor = env.bounded(child.wait_for_generation(2)).unwrap(); assert_eq!(env.bounded(actor.request(Ask)).unwrap(), 1); let snapshot = child.snapshot(); - assert_eq!(snapshot.generation, 2, "{place:?} in_future={in_future}"); + assert_eq!(snapshot.generation, 3, "{place:?} in_future={in_future}"); assert!(snapshot .last_error .as_deref() diff --git a/concurrency/tests/process_failures.rs b/concurrency/tests/process_failures.rs index 64812b6ca..2967c30ba 100644 --- a/concurrency/tests/process_failures.rs +++ b/concurrency/tests/process_failures.rs @@ -30,17 +30,24 @@ impl Drop for Fixture { } } impl Handler for Fixture { - async fn handle(&mut self, _: Crash, _: &Context) { - std::fs::write( - self.path.join("effect"), - b"effect happened before process failure", - ) - .unwrap(); - if self.abort { - std::process::abort(); - } - loop { - std::thread::park(); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + std::fs::write( + self.path.join("effect"), + b"effect happened before process failure", + ) + .unwrap(); + if self.abort { + std::process::abort(); + } + loop { + std::thread::park(); + } } } } diff --git a/concurrency/tests/protocol_timeouts.rs b/concurrency/tests/protocol_timeouts.rs index c9f491425..5fc61f449 100644 --- a/concurrency/tests/protocol_timeouts.rs +++ b/concurrency/tests/protocol_timeouts.rs @@ -58,21 +58,45 @@ mod asynchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Work, _: &Context) -> u64 { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis + async fn handle( + &mut self, + msg: timed_protocol::Work, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; + reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: timed_protocol::Brief, _: &Context) -> u64 { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis + async fn handle( + &mut self, + msg: timed_protocol::Brief, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; + reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, msg: legacy_protocol::Legacy, _: &Context) -> u64 { - spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; - msg.millis + async fn handle( + &mut self, + msg: legacy_protocol::Legacy, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + }; + reply.reply(__reply_value); } } // Tokio's paused clock normally; the simulator's clock is always virtual. @@ -130,15 +154,31 @@ mod synchronous { struct Worker; impl Actor for Worker {} impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Work, _: &Context) -> u64 { - std::thread::sleep(Duration::from_millis(msg.millis)); - msg.millis + fn handle( + &mut self, + msg: timed_protocol::Work, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + std::thread::sleep(Duration::from_millis(msg.millis)); + msg.millis + }; + reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, msg: timed_protocol::Brief, _: &Context) -> u64 { - std::thread::sleep(Duration::from_millis(msg.millis)); - msg.millis + fn handle( + &mut self, + msg: timed_protocol::Brief, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + std::thread::sleep(Duration::from_millis(msg.millis)); + msg.millis + }; + reply.reply(__reply_value); } } #[test] @@ -167,26 +207,53 @@ mod synchronous { } } impl Handler for Held { - fn handle(&mut self, _: zero_protocol::Zero, _: &Context) {} + fn handle( + &mut self, + _: zero_protocol::Zero, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Timeout, _: &Context) -> u64 { - 31 + fn handle( + &mut self, + _: naming_protocol::Timeout, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(31); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::ActorRef, _: &Context) -> u64 { - 32 + fn handle( + &mut self, + _: naming_protocol::ActorRef, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(32); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::TimeoutOverride, _: &Context) -> u64 { - 33 + fn handle( + &mut self, + _: naming_protocol::TimeoutOverride, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(33); } } impl Handler for Worker { - fn handle(&mut self, _: naming_protocol::Dispatch0, _: &Context) -> u64 { - 34 + fn handle( + &mut self, + _: naming_protocol::Dispatch0, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(34); } } #[test] @@ -205,8 +272,13 @@ mod synchronous { actor.join(); } impl Handler for Worker { - fn handle(&mut self, msg: owned_protocol::Owned, _: &Context) -> u64 { - msg.__spawned_request_timeout.value + fn handle( + &mut self, + msg: owned_protocol::Owned, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(msg.__spawned_request_timeout.value); } } #[test] diff --git a/concurrency/tests/registry_turn_loom.rs b/concurrency/tests/registry_turn_loom.rs new file mode 100644 index 000000000..74687a9a5 --- /dev/null +++ b/concurrency/tests/registry_turn_loom.rs @@ -0,0 +1,136 @@ +//! Loom model of the owner's turn phase and in-turn drop commit +//! (`src/reply/turn.rs`, docs/design/deferred-replies.md ยง4): drops from +//! other threads racing turn end, exit and death. Every dropped obligation +//! gets exactly one outcome, `Stopped` exactly when the turn it landed on +//! panicked or the owner was exiting or dead, and none is lost. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicU8, Ordering}; + pub(crate) use loom::sync::Mutex; +} + +#[allow(dead_code)] +#[path = "../src/reply/turn.rs"] +mod turn; + +use loom::sync::Arc; +use std::sync::Mutex as StdMutex; +use turn::{Outcome, TurnPhase}; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.check(f); +} + +/// Records each item's outcome; panics on a second one. +#[derive(Default)] +struct Outcomes(StdMutex>); +impl Outcomes { + fn apply(&self, items: Vec, outcome: Outcome) { + for item in items { + self.one(item, outcome); + } + } + fn one(&self, item: u32, outcome: Outcome) { + let mut seen = self.0.lock().unwrap(); + assert!( + seen.iter().all(|(i, _)| *i != item), + "item {item} answered twice" + ); + seen.push((item, outcome)); + } + fn of(&self, item: u32) -> Option { + self.0 + .lock() + .unwrap() + .iter() + .find(|(i, _)| *i == item) + .map(|(_, o)| *o) + } +} + +fn drop_on(turn: &TurnPhase, outcomes: &Outcomes, item: u32) { + if let Some((item, outcome)) = turn.dropped(item) { + outcomes.one(item, outcome); + } +} + +/// A drop from another thread racing the end of a turn that returned or +/// panicked: it commits with that turn, or (if after it) answers `Dropped`. +#[test] +fn drop_vs_turn_end() { + for panicked in [false, true] { + model(move || { + let turn = Arc::new(TurnPhase::::default()); + let outcomes = Arc::new(Outcomes::default()); + turn.begin_turn(); + let dropper = { + let (turn, outcomes) = (turn.clone(), outcomes.clone()); + loom::thread::spawn(move || drop_on(&turn, &outcomes, 1)) + }; + let (drops, outcome) = turn.end_turn(panicked); + let committed = !drops.is_empty(); + outcomes.apply(drops, outcome); + dropper.join().unwrap(); + let got = outcomes.of(1).expect("a dropped obligation was lost"); + if committed { + let expected = if panicked { + Outcome::Stopped + } else { + Outcome::Dropped + }; + assert_eq!(got, expected); + } else { + assert_eq!( + got, + Outcome::Dropped, + "a drop after the turn answers at once" + ); + } + }); + } +} + +/// Drops racing exit and death: nothing is lost, and anything recorded +/// on the interrupted turn commits as `Stopped`. +#[test] +fn drops_vs_exit_vs_death() { + model(|| { + let turn = Arc::new(TurnPhase::::default()); + let outcomes = Arc::new(Outcomes::default()); + turn.begin_turn(); + let droppers: Vec<_> = (1..=2) + .map(|item| { + let (turn, outcomes) = (turn.clone(), outcomes.clone()); + loom::thread::spawn(move || drop_on(&turn, &outcomes, item)) + }) + .collect(); + outcomes.apply(turn.exiting(), Outcome::Stopped); + outcomes.apply(turn.close(), Outcome::Stopped); + for dropper in droppers { + dropper.join().unwrap(); + } + for item in 1..=2 { + assert_eq!(outcomes.of(item), Some(Outcome::Stopped), "item {item}"); + } + assert_eq!(turn.load(), turn::DEAD); + }); +} + +/// Exit is idempotent and never revives a dead owner. +#[test] +fn exiting_after_death_is_a_no_op() { + model(|| { + let turn = Arc::new(TurnPhase::::default()); + let late = { + let turn = turn.clone(); + loom::thread::spawn(move || turn.exiting()) + }; + let closed = turn.close(); + assert!(closed.is_empty()); + assert!(late.join().unwrap().is_empty()); + assert_eq!(turn.load(), turn::DEAD); + }); +} diff --git a/concurrency/tests/reply_slot_loom.rs b/concurrency/tests/reply_slot_loom.rs new file mode 100644 index 000000000..75faef1f7 --- /dev/null +++ b/concurrency/tests/reply_slot_loom.rs @@ -0,0 +1,715 @@ +//! Loom model of the reply slot's packed-word protocol +//! (docs/design/deferred-replies.md ยง1). One test per concurrent pair: +//! reply, expiry, shortening, abandonment, token drop, owner close, defer, +//! linked-job attach and end, and the parked/polled receiver. +//! +//! Run: `RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test -p spawned-concurrency +//! --no-default-features --test reply_slot_loom --release`. +#![cfg(spawned_scheduler_loom)] + +mod sync { + pub(crate) use loom::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + pub(crate) use loom::sync::{Arc, Mutex, MutexGuard}; + pub(crate) type SlotOwner = Option<()>; + pub(crate) type Reason = (); + pub(crate) fn section(f: impl FnOnce() -> T) -> T { + f() + } + pub(crate) fn left_open(_: &SlotOwner, _: usize, _: &'static str, _: bool) {} + pub(crate) fn shortened(_: &SlotOwner, _: usize) {} + pub(crate) fn parked() {} + + /// The thread's park token: `std::thread::park`'s semantics (a wake + /// sets the token; a park consumes it), modelled with a Loom mutex and + /// condvar because Loom's own `unpark` also wakes a thread blocked in a + /// `join`. + #[derive(Default)] + pub(crate) struct Token { + set: loom::sync::Mutex, + cv: loom::sync::Condvar, + } + impl std::task::Wake for Token { + fn wake(self: std::sync::Arc) { + *self.set.lock().unwrap() = true; + self.cv.notify_all(); + } + } + loom::thread_local! { + static TOKEN: std::sync::Arc = std::sync::Arc::new(Token::default()); + } + + /// A waker that unparks the current Loom thread. + pub(crate) fn with_thread_waker(f: impl FnOnce(&std::task::Waker) -> T) -> T { + let waker = TOKEN.with(|token| std::task::Waker::from(token.clone())); + f(&waker) + } + + /// Loom has no timed park; tests park only on slots without a deadline. + pub(crate) fn park(ticks: Option) { + assert!(ticks.is_none(), "timed park is not modelled"); + let token = TOKEN.with(|token| token.clone()); + let mut set = token.set.lock().unwrap(); + while !*set { + set = token.cv.wait(set).unwrap(); + } + *set = false; + } +} + +#[allow(dead_code)] +#[path = "../src/reply/slot.rs"] +mod slot; + +use loom::sync::atomic::{AtomicU64, Ordering}; +use loom::sync::Arc; +use slot::{credits, jobs, state, CreditPool, Delivery, Refused, Slot, Terminal, NEVER}; + +fn model(f: impl Fn() + Sync + Send + 'static) { + let mut builder = loom::model::Builder::new(); + builder.preemption_bound = Some(3); + builder.max_permutations = Some(200_000); + builder.check(f); +} + +/// A model clock: ticks advanced explicitly by the test. +#[derive(Clone)] +struct Clock(Arc); +impl Clock { + fn at(t: u64) -> Self { + Self(Arc::new(AtomicU64::new(t))) + } + fn now(&self) -> impl Fn() -> u64 + '_ { + move || self.0.load(Ordering::SeqCst) + } + fn set(&self, t: u64) { + self.0.store(t, Ordering::SeqCst); + } +} + +/// The pool's count always equals the sum of `credits(word)` over its slots +/// once every thread has joined. +fn assert_balanced(pool: &CreditPool, slots: &[&Slot]) { + let held: u64 = slots.iter().map(|s| credits(s.word())).sum(); + assert_eq!( + pool.used() as u64, + held, + "credit counter diverged from slot words" + ); + assert!(pool.used() <= pool.max(), "credit bound exceeded"); +} + +fn deferred(pool: &Arc, deadline: u64) -> Arc> { + let slot = Arc::new(Slot::new(deadline)); + slot.defer(pool, &|| 0).expect("defer on a fresh slot"); + slot +} + +/// Reply versus expiry as the clock reaches the deadline (OTP: the +/// caller's expiry is the only timeout authority; the reply reads no +/// clock). Exactly one wins the word: a reply that wins is delivered even +/// at the deadline (before the expiry ran); one that loses gets its value +/// back as `Expired`, and the caller sees `TimedOut`, never the value. +#[test] +fn reply_vs_expiry() { + model(|| { + let clock = Clock::at(0); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(7)) + }; + clock.set(10); + let expired = slot.check_deadline(&clock.now()); + let (delivery, back) = replier.join().unwrap(); + assert!(expired, "the deadline has passed"); + match delivery { + Delivery::Delivered => { + assert_eq!(state(slot.word()), slot::REPLIED); + assert!(back.is_none()); + assert_eq!(slot.try_take(&clock.now()), Some(Ok(7))); + } + Delivery::Expired => { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert_eq!(back, Some(7)); + assert_eq!(slot.try_take(&clock.now()), Some(Err(Terminal::TimedOut))); + } + other => panic!("unexpected {other:?}"), + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// Reply versus `shorten_deadline` into the past: the shortening's +/// `[deadline]` check expires the slot, racing the reply on the word. The +/// reply wins (delivered, whatever the deadline became) or loses +/// (`Expired`, and then the shortening was applied). +#[test] +fn reply_vs_shortening() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1)) + }; + let lowered = slot.shorten(3, &clock.now()); + let (delivery, _) = replier.join().unwrap(); + match delivery { + Delivery::Delivered => assert_eq!(state(slot.word()), slot::REPLIED), + Delivery::Expired => { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert!(lowered); + assert_eq!(slot.deadline(), 3); + } + other => panic!("unexpected {other:?}"), + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// Reply versus the caller dropping its receiver. +#[test] +fn reply_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1)) + }; + let abandoned = slot.abandon(); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(abandoned, delivery == Delivery::Delivered); + if abandoned { + assert_eq!(state(slot.word()), slot::ABANDONED); + assert_eq!((delivery, back), (Delivery::CallerGone, Some(1))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// The in-turn reply (one CAS `OPEN -> REPLIED`, value written under the +/// slot lock) versus the caller abandoning: exactly one wins; a lost reply +/// gets its value back. +#[test] +fn reply_in_turn_vs_abandon() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(1)) + }; + let abandoned = slot.abandon(); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(abandoned, delivery == Delivery::Delivered); + if abandoned { + assert_eq!(state(slot.word()), slot::ABANDONED); + assert_eq!((delivery, back), (Delivery::CallerGone, Some(1))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + assert_eq!(back, None); + } + }); +} + +/// The in-turn reply versus the caller's timer expiring the slot: one wins; +/// an expired slot hands the value back as `Expired`. +#[test] +fn reply_in_turn_vs_expiry() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(10)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(7)) + }; + let expired = slot.terminate(Terminal::TimedOut); + let (delivery, back) = replier.join().unwrap(); + assert_ne!(expired, delivery == Delivery::Delivered); + if expired { + assert_eq!(state(slot.word()), slot::TIMED_OUT); + assert_eq!((delivery, back), (Delivery::Expired, Some(7))); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + } + }); +} + +/// A polled receiver sees the in-turn reply's value (written before the +/// publishing CAS, under the lock the receiver takes). +#[test] +fn polled_receiver_vs_reply_in_turn() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(4).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Ok(4)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// Reply versus the owner's registry close at death. A close that finds +/// `WRITING` waits for the reply to resolve. +#[test] +fn reply_vs_owner_close() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(1).0) + }; + let closed = slot.close(); + assert!( + slot.is_terminal(), + "close returned while a reply was WRITING" + ); + let delivery = replier.join().unwrap(); + if closed { + assert_eq!(state(slot.word()), slot::STOPPED); + assert_eq!(delivery, Delivery::OwnerExited); + } else { + assert_eq!(state(slot.word()), slot::REPLIED); + assert_eq!(delivery, Delivery::Delivered); + } + assert_balanced(&pool, &[&slot]); + }); +} + +/// A held token dropped (`DROPPED`) versus the owner close (`STOPPED`) versus +/// the deadline heap expiring it: exactly one wins, the credit returns once. +#[test] +fn drop_vs_close_vs_expiry() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 10); + let dropper = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let expirer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.check_deadline(&|| 10)) + }; + let closed = slot.close(); + let dropped = dropper.join().unwrap(); + assert!(expirer.join().unwrap()); + let s = state(slot.word()); + let expired = s == slot::TIMED_OUT; + assert_eq!([closed, dropped, expired].iter().filter(|b| **b).count(), 1); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// `defer()` versus the caller abandoning: a slot that went terminal never +/// acquires a credit, and a deferred one releases it on abandonment. +#[test] +fn defer_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let caller = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.abandon()) + }; + let deferred = slot.defer(&pool, &|| 0); + caller.join().unwrap(); + assert_eq!(state(slot.word()), slot::ABANDONED); + match deferred { + Ok(()) | Err(Refused::Closed) => {} + Err(Refused::Overloaded) => panic!("bound of one exceeded by one token"), + } + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// Two slots defer concurrently against a bound of one: one wins, the other's +/// caller is answered `Overloaded` in the same step. +#[test] +fn defer_at_the_bound() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let a: Arc> = Arc::new(Slot::new(NEVER)); + let b: Arc> = Arc::new(Slot::new(NEVER)); + let other = { + let (b, pool) = (b.clone(), pool.clone()); + loom::thread::spawn(move || b.defer(&pool, &|| 0)) + }; + let first = a.defer(&pool, &|| 0); + let second = other.join().unwrap(); + let outcomes = [first, second]; + assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1); + let loser = if first.is_ok() { &b } else { &a }; + assert_eq!(state(loser.word()), slot::OVERLOADED); + assert_balanced(&pool, &[&a, &b]); + }); +} + +/// Linked jobs: a second attach, both job ends, and the owner close race. +/// The token and its first job share one credit; credits always match +/// `max(JOBS, live token)` and return to zero. +#[test] +fn attach_vs_job_end_vs_close() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0) + .expect("first attach shares the token's credit"); + assert_eq!(pool.used(), 1); + let second = { + let slot = slot.clone(); + loom::thread::spawn(move || { + if slot.attach(&|| 0).is_ok() { + slot.job_end(); + } + }) + }; + let first_end = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.job_end()) + }; + slot.close(); + second.join().unwrap(); + first_end.join().unwrap(); + assert_eq!(jobs(slot.word()), 0); + assert_eq!(state(slot.word()), slot::STOPPED); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A job ending while the reply moves `WRITING -> REPLIED`: the state-field +/// CAS must not lose the `JOBS` decrement. +#[test] +fn job_end_during_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0).unwrap(); + slot.attach(&|| 0).unwrap(); + assert_eq!(pool.used(), 2); + let ender = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.job_end()) + }; + assert_eq!(slot.reply(9).0, Delivery::Delivered); + ender.join().unwrap(); + assert_eq!(jobs(slot.word()), 1); + assert_eq!(pool.used(), 1, "the job still running keeps its credit"); + slot.job_end(); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A further attach while a job still runs after the token closed is +/// refused (`Closed`), and the running job's credit survives the close. +#[test] +fn attach_after_close_is_refused() { + model(|| { + let pool = Arc::new(CreditPool::new(2)); + let slot = deferred(&pool, NEVER); + slot.attach(&|| 0).unwrap(); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let attached = slot.attach(&|| 0); + closer.join().unwrap(); + match attached { + Ok(()) => { + assert_eq!(jobs(slot.word()), 2); + assert_eq!(pool.used(), 2); + slot.job_end(); + } + Err(refused) => assert_eq!(refused, Refused::Closed), + } + assert_eq!(pool.used(), 1); + slot.job_end(); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A parked threads-mode receiver is woken by the reply (no lost wakeup). +#[test] +fn parked_receiver_vs_reply() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(3).0) + }; + assert_eq!(slot.recv_blocking(&|| 0), Ok(3)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// A parked receiver is woken by the owner close. +#[test] +fn parked_receiver_vs_close() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.close()) + }; + assert_eq!(slot.recv_blocking(&|| 0), Err(Terminal::Stopped)); + assert!(closer.join().unwrap()); + }); +} + +/// A polled (async) receiver is woken by the reply (no lost wakeup). +#[test] +fn polled_receiver_vs_reply() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(4).0) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Ok(4)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// A polled receiver woken by a drop of the deferred token. +#[test] +fn polled_receiver_vs_token_drop() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let dropper = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.terminate(Terminal::Dropped)) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Err(Terminal::Dropped)); + assert!(dropper.join().unwrap()); + assert_balanced(&pool, &[&slot]); + }); +} + +/// Caller give-up (abandon) versus deadline shortening versus the heap's +/// expiry: one terminal state and a single credit return. +#[test] +fn abandon_vs_shorten_vs_expiry() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let shortener = { + let (slot, clock) = (slot.clone(), clock.clone()); + loom::thread::spawn(move || { + slot.shorten(5, &clock.now()); + }) + }; + let abandoned = slot.abandon(); + shortener.join().unwrap(); + let s = state(slot.word()); + assert!(s == slot::ABANDONED || s == slot::TIMED_OUT); + assert_eq!(abandoned, s == slot::ABANDONED); + assert_eq!(pool.used(), 0); + assert_balanced(&pool, &[&slot]); + }); +} + +/// agy #6: a deferral that finds the pool full while the caller abandons +/// reports `Overloaded` only if the slot really became `OVERLOADED`; a slot +/// that closed first reports `Closed`. +#[test] +fn defer_at_the_bound_vs_abandon() { + model(|| { + let pool = Arc::new(CreditPool::new(0)); + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let caller = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.abandon()) + }; + let deferred = slot.defer(&pool, &|| 0); + let abandoned = caller.join().unwrap(); + match deferred { + Err(Refused::Overloaded) => { + assert_eq!(state(slot.word()), slot::OVERLOADED); + assert!(!abandoned); + } + Err(Refused::Closed) => assert!(abandoned), + Ok(()) => panic!("a full pool granted a credit"), + } + assert_eq!(pool.used(), 0); + }); +} + +/// A parked threads-mode receiver is woken by the in-turn reply (one CAS +/// publishing the value; the receiver's unparker sits in the waker cell). +#[test] +fn parked_receiver_vs_reply_in_turn() { + model(|| { + let slot: Arc> = Arc::new(Slot::new(NEVER)); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply_in_turn(5).0) + }; + assert_eq!(slot.recv_blocking(&|| 0), Ok(5)); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + }); +} + +/// The death point's batched close: the close collects the wake-up, the +/// parked receivers are notified first (`notify_parked`), the rest runs +/// after. A parked receiver is never lost between the two steps. +#[test] +fn parked_receiver_vs_batched_close() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let closer = { + let slot = slot.clone(); + loom::thread::spawn(move || { + let mut wake = slot.close_collect().expect("the close wins an open slot"); + slot.notify_parked(&mut wake); + slot.run_wake(wake); + }) + }; + assert_eq!(slot.recv_blocking(&|| 0), Err(Terminal::Stopped)); + closer.join().unwrap(); + assert_eq!(pool.used(), 0, "the credit returned with the close"); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A token watcher (`ReplyTo::closed`, caller-gone job cancellation) is +/// woken by a deferred reply from another thread (no lost wakeup), and the +/// caller's receiver gets the value. +#[test] +fn watcher_and_receiver_vs_deferred_reply() { + model(|| { + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, NEVER); + let replier = { + let slot = slot.clone(); + loom::thread::spawn(move || slot.reply(8).0) + }; + loom::future::block_on(std::future::poll_fn(|cx| { + slot.poll_closed(cx.waker(), &|| 0) + })); + assert_eq!(replier.join().unwrap(), Delivery::Delivered); + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Ok(8)); + assert_balanced(&pool, &[&slot]); + }); +} + +/// A polled receiver versus a shortening (which wakes it to re-arm) and +/// the expiry that follows: the receiver ends `TimedOut` or with the +/// reply, never blocked on a taken waker. +#[test] +fn polled_receiver_vs_shorten_and_reply() { + model(|| { + let clock = Clock::at(5); + let pool = Arc::new(CreditPool::new(1)); + let slot = deferred(&pool, 100); + let owner = { + let (slot, clock) = (slot.clone(), clock.clone()); + loom::thread::spawn(move || { + slot.shorten(5, &clock.now()); + slot.reply(2).0 + }) + }; + let got = loom::future::block_on(std::future::poll_fn(|cx| slot.poll(cx.waker()))); + assert_eq!(got, Err(Terminal::TimedOut)); + assert_eq!(owner.join().unwrap(), Delivery::Expired); + assert_balanced(&pool, &[&slot]); + }); +} + +/// The dispatching envelope lends its slot to the turn (no reference taken +/// per message): the in-turn reply goes through the borrow while the +/// caller takes the value or abandons, and either side may drop the last +/// reference. The value is dropped exactly once, by whoever holds it last, +/// and never while the other side still reads it. +#[test] +fn lent_reply_vs_receiver_last_reference() { + use loom::sync::atomic::AtomicUsize; + struct Counted(Arc); + impl Drop for Counted { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + model(|| { + let drops = Arc::new(AtomicUsize::new(0)); + let lent: Arc> = Arc::new(Slot::new(NEVER)); + let receiver = { + let slot = lent.clone(); + loom::thread::spawn(move || match slot.try_take(&|| 0) { + Some(Ok(value)) => { + drop(value); + true + } + Some(Err(_)) => false, + None => { + slot.abandon(); + false + } + }) + }; + let borrowed: &Arc> = &lent; + let (delivery, back) = borrowed.reply_in_turn(Counted(drops.clone())); + drop(back); + drop(lent); + let took = receiver.join().unwrap(); + assert_eq!( + drops.load(Ordering::SeqCst), + 1, + "the value is dropped exactly once" + ); + if took { + assert_eq!(delivery, Delivery::Delivered); + } + }); +} + +/// The caller takes a reply without the slot lock once it owns the slot +/// alone (`Arc::try_unwrap`, then `into_reply`), while the owner's turn +/// replies through its lent reference and drops it: the value is the +/// replied one, taken exactly once, by the lock-free path or the locked one. +#[test] +fn unshared_take_vs_lent_reply_and_release() { + model(|| { + let lent: Arc> = Arc::new(Slot::new(NEVER)); + let mut caller = Some(lent.clone()); + let owner = loom::thread::spawn(move || { + let (delivery, back) = lent.reply_in_turn(6); + drop(lent); + (delivery, back) + }); + let got = loop { + let shared = caller.take().expect("the caller holds its slot"); + if state(shared.word()) == slot::REPLIED && Arc::strong_count(&shared) == 1 { + match Arc::try_unwrap(shared) { + Ok(owned) => break owned.into_reply(), + Err(shared) => caller = Some(shared), + } + } else { + caller = Some(shared); + } + let shared = caller.as_ref().expect("put back"); + if let Some(result) = shared.try_take(&|| 0) { + break result.ok(); + } + loom::thread::yield_now(); + }; + assert_eq!(got, Some(6)); + assert_eq!(owner.join().unwrap(), (Delivery::Delivered, None)); + }); +} diff --git a/concurrency/tests/reply_trace.rs b/concurrency/tests/reply_trace.rs new file mode 100644 index 000000000..7159647e0 --- /dev/null +++ b/concurrency/tests/reply_trace.rs @@ -0,0 +1,798 @@ +//! Records reply-slot traces from real actors for TLA+ trace validation +//! against `verification/tla/DeferredReplies.tla` (verification/tla/README.md). +//! Each scenario writes `replies_.json` to `$LIFECYCLE_TRACE_DIR` +//! (default `target/lifecycle-traces`); `verification/tla/trace_check.py` +//! checks them (schema `replies-v1`). +#![cfg(feature = "lifecycle-trace")] + +use spawned_concurrency::{ + lifecycle_trace, + message::Message, + supervision::{ChildSpec, SupervisorConfig}, + system::ActorSystem, + tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, JobError, JobOptions, JobOutcome, + PendingReply, ReplyTo, + }, + ActorError, ExitReason, +}; +use spawned_rt::tasks::sleep; +use std::{path::PathBuf, time::Duration}; + +const BOUND: usize = 1; + +fn trace_dir() -> PathBuf { + std::env::var_os("LIFECYCLE_TRACE_DIR") + .map(PathBuf::from) + .unwrap_or_else(|| { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../target/lifecycle-traces") + }) +} + +#[derive(Default)] +struct Owner { + stored: Vec>, + /// The downstream actor B its jobs call. + backend: Option>, +} + +impl Actor for Owner { + const MAX_OUTSTANDING: usize = BOUND; +} + +/// Replied in its turn. +struct Echo(u32); +impl Message for Echo { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, msg: Echo, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} + +/// Deferred and stored for a later turn (`Overloaded` at the bound). +struct Hold; +impl Message for Hold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Hold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.stored.push(token); + } + } +} + +/// Never answered: the unused obligation answers `ReplyDropped` at turn end. +struct Ignore; +impl Message for Ignore { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Ignore, _reply: PendingReply<'_, u32>, _: &Context) {} +} + +/// Deferred, then the turn panics: the token is held until the death point, +/// which answers it `ActorStopped(Panic(..))`. +struct Boom; +impl Message for Boom { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: Boom, reply: PendingReply<'_, u32>, _: &Context) { + let _token = reply.defer(); + panic!("boom"); + } +} + +/// A sent (not requested) message: answer, or drop, every stored token. +enum Flush { + Reply, + Drop, +} +impl Message for Flush { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Flush, _: PendingReply<'_, ()>, _: &Context) { + for token in self.stored.drain(..) { + match msg { + Flush::Reply => { + token.reply(7); + } + Flush::Drop => drop(token), + } + } + } +} + +/// Wait until every message sent before has been handled. +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +/// Deferred, its deadline shortened, then stored. +struct ShortHold; +impl Message for ShortHold { + type Result = u32; +} +impl Handler for Owner { + async fn handle(&mut self, _: ShortHold, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + token.shorten_deadline(Duration::from_secs(3)); + self.stored.push(token); + } + } +} + +/// Deferred into a `pipe_reply` job: `work_ms` of work under an optional job +/// deadline (`deadline_ms`), optionally cancelled when the caller gives up; +/// with `downstream`, the work asks the backend B (`Ask`, or `AskNow`). +struct Piped { + work_ms: u64, + deadline_ms: Option, + cancel_on_gone: bool, + downstream: Option, +} +impl Message for Piped { + type Result = u32; +} +/// The job's notification, with the token back. +struct JobDone { + res: JobOutcome, + token: ReplyTo, +} +impl Message for JobDone { + type Result = (); +} +impl Handler for Owner { + async fn handle(&mut self, msg: Piped, reply: PendingReply<'_, u32>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let mut opts = JobOptions::new(); + if let Some(ms) = msg.deadline_ms { + opts = opts.deadline(Duration::from_millis(ms)); + } + if msg.cancel_on_gone { + opts = opts.cancel_on_caller_gone(); + } + let backend = msg + .downstream + .map(|later| (later, self.backend.clone().unwrap())); + let work_ms = msg.work_ms; + let piped = ctx.pipe_reply( + opts, + token, + move |_| async move { + let got = match backend { + Some((true, b)) => b.request(Ask).await.unwrap_or(0), + Some((false, b)) => b.request(AskNow).await.unwrap_or(0), + None => 5, + }; + sleep(Duration::from_millis(work_ms)).await; + got + }, + |res, token| JobDone { res, token }, + ); + if let Err(e) = piped { + drop(e.into_token()); + } + } +} +impl Handler for Owner { + async fn handle(&mut self, msg: JobDone, _: PendingReply<'_, ()>, ctx: &Context) { + match msg.res { + Ok(v) => { + msg.token.reply(v); + } + Err(JobError::DeadlineExceeded) => { + // The work still runs and holds the request's credit: a + // second job needs a fresh one, and BOUND = 1 refuses it. + let again = ctx.pipe_reply( + JobOptions::new(), + msg.token, + |_| async { 1u32 }, + |res, token| JobDone { res, token }, + ); + match again { + Ok(_) => panic!("a second job found a free credit"), + Err(e) => { + e.into_token().reply(9); + } + } + } + Err(_) => drop(msg.token), + } + } +} + +/// The downstream actor B: defers `Ask` until `Release`, answers `AskNow`. +#[derive(Default)] +struct Backend { + held: Vec>, +} +impl Actor for Backend { + const MAX_OUTSTANDING: usize = 1; +} +struct Ask; +impl Message for Ask { + type Result = u32; +} +struct AskNow; +impl Message for AskNow { + type Result = u32; +} +struct Release; +impl Message for Release { + type Result = (); +} +struct Held; +impl Message for Held { + type Result = usize; +} +impl Handler for Backend { + async fn handle(&mut self, _: Ask, reply: PendingReply<'_, u32>, _: &Context) { + if let Ok(token) = reply.defer() { + self.held.push(token); + } + } +} +impl Handler for Backend { + async fn handle(&mut self, _: AskNow, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(4); + } +} +impl Handler for Backend { + async fn handle(&mut self, _: Release, _: PendingReply<'_, ()>, _: &Context) { + for token in self.held.drain(..) { + token.reply(8); + } + } +} +impl Handler for Backend { + async fn handle(&mut self, _: Held, reply: PendingReply<'_, usize>, _: &Context) { + reply.reply(self.held.len()); + } +} + +/// Start an owner whose reply slots are traced. +fn start() -> ActorRef { + let actor = Owner::default().start(); + lifecycle_trace::watch_deferred(id(&actor)); + actor +} + +fn id(actor: &ActorRef) -> u64 { + actor.id().to_string().parse().unwrap() +} + +/// Write the owner's reply trace once it is dead. +fn write(name: &str, owner: u64) -> Vec { + let events = lifecycle_trace::take_replies(owner); + assert!( + events.iter().any(|e| e.action == "Death"), + "{name}: the owner's death was not recorded" + ); + let json = format!( + "{{\"name\":\"replies_{name}\",\"schema\":\"replies-v1\",\"bound\":{BOUND},\"events\":[\n{}\n]}}\n", + events + .iter() + .map(|e| e.to_json()) + .collect::>() + .join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("replies_{name}.json")), json).unwrap(); + events + .iter() + .map(|e| { + format!( + "{}{}", + e.action, + if e.x.is_empty() { + String::new() + } else { + format!(":{}", e.x) + } + ) + }) + .collect() +} + +async fn stop(name: &str, actor: ActorRef) -> Vec { + let owner = id(&actor); + actor.context().stop(); + actor.join().await; + write(name, owner) +} + +#[tokio::test] +async fn reply_in_turn() { + let actor = start(); + let got = actor.request(Echo(3)).await; + assert!(matches!(got, Ok(3)), "{got:?}"); + let events = stop("in_turn", actor).await; + assert_eq!( + events, + ["Send", "Dispatch", "Discharge:ok", "Exit", "Death"] + ); +} + +#[tokio::test] +async fn stored_token_answered_later() { + let actor = start(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Reply).unwrap(); + let got = held.await; + assert!(matches!(got, Ok(7)), "{got:?}"); + stop("later_reply", actor).await; +} + +#[tokio::test] +async fn stored_token_dropped_later() { + let actor = start(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Drop).unwrap(); + let got = held.await; + assert!(matches!(got, Err(ActorError::ReplyDropped)), "{got:?}"); + stop("later_drop", actor).await; +} + +#[tokio::test] +async fn unused_obligation_answers_dropped() { + let actor = start(); + let got = actor.request(Ignore).await; + assert!(matches!(got, Err(ActorError::ReplyDropped)), "{got:?}"); + stop("unused", actor).await; +} + +#[tokio::test] +async fn defer_at_the_bound_is_overloaded() { + let actor = start(); + let first = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let got = actor.request(Hold).await; + assert!(matches!(got, Err(ActorError::Overloaded)), "{got:?}"); + actor.send(Flush::Reply).unwrap(); + let got = first.await; + assert!(matches!(got, Ok(7)), "{got:?}"); + stop("overloaded", actor).await; +} + +#[tokio::test] +async fn stored_token_expires_at_its_deadline() { + let actor = start(); + let deadline = Deadline::At(spawned_rt::time::Instant::now() + Duration::from_millis(30)); + let held = actor.send_request(Hold, deadline).unwrap(); + let got = held.await; + assert!(matches!(got, Err(ActorError::RequestTimeout)), "{got:?}"); + actor.send(Flush::Reply).unwrap(); + actor.request(Sync).await.unwrap(); + stop("expired", actor).await; +} + +#[tokio::test] +async fn caller_gives_up() { + let actor = start(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + drop(held); + actor.send(Flush::Reply).unwrap(); + actor.request(Sync).await.unwrap(); + stop("abandoned", actor).await; +} + +#[tokio::test] +async fn death_closes_stored_tokens() { + let actor = start(); + let held = actor + .send_request(Hold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + let owner = id(&actor); + actor.context().stop(); + let got = held.await; + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Normal))), + "{got:?}" + ); + actor.join().await; + let events = write("death", owner); + assert!( + !events.iter().any(|e| e == "Discharge:stopped"), + "{events:?}" + ); +} + +#[tokio::test] +async fn panic_in_turn_answers_stopped() { + let actor = start(); + // The token unwound by the panic is exit pending: answered at the death + // point with the final reason, not by a discharge of its own. + let got = actor.request(Boom).await; + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), + "{got:?}" + ); + actor.join().await; + sleep(Duration::from_millis(10)).await; + write("panic", id(&actor)); +} + +#[tokio::test] +async fn shortened_deadline_then_reply() { + let actor = start(); + let held = actor + .send_request(ShortHold, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + actor.send(Flush::Reply).unwrap(); + assert!(matches!(held.await, Ok(7))); + let events = stop("shorten", actor).await; + in_order( + &events, + &["Dispatch", "Shorten", "Discharge:ok", "Exit", "Death"], + ); +} + +/// `expected` occurs in `events` in this order. +fn in_order(events: &[String], expected: &[&str]) { + let mut rest = events.iter(); + for want in expected { + assert!( + rest.any(|e| e == want), + "{want} missing or out of order: {events:?}" + ); + } +} + +fn piped(work_ms: u64) -> Piped { + Piped { + work_ms, + deadline_ms: None, + cancel_on_gone: false, + downstream: None, + } +} + +#[tokio::test] +async fn job_result_answers_the_caller() { + let actor = start(); + assert!(matches!(actor.request(piped(1)).await, Ok(5))); + let events = stop("job_done", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn job_deadline_hands_the_token_back() { + let actor = start(); + let got = actor + .request(Piped { + deadline_ms: Some(20), + ..piped(150) + }) + .await; + assert!(matches!(got, Ok(9)), "{got:?}"); + // The work runs on past its deadline, then ends quietly. + sleep(Duration::from_millis(250)).await; + let events = stop("job_deadline", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "JobNotify:deadline", + "PipeReply:refused", + "Discharge:ok", + "JobEnd", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn caller_gone_cancels_the_job() { + let actor = start(); + let held = actor + .send_request( + Piped { + cancel_on_gone: true, + ..piped(10_000) + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + actor.request(Sync).await.unwrap(); + drop(held); + sleep(Duration::from_millis(50)).await; + let events = stop("job_cancelled", actor).await; + in_order( + &events, + &[ + "PipeReply:linked", + "Discharge:abandoned", + "JobCancel", + "JobEnd", + "Exit", + "Death", + ], + ); +} + +#[tokio::test] +async fn owner_exit_cancels_the_job() { + let actor = start(); + let held = actor + .send_request(piped(10_000), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + actor.request(Sync).await.unwrap(); + let owner = id(&actor); + actor.context().stop(); + let got = held.await; + assert!( + matches!(got, Err(ActorError::ActorStopped(ExitReason::Normal))), + "{got:?}" + ); + actor.join().await; + sleep(Duration::from_millis(50)).await; + let events = write("job_exit", owner); + in_order(&events, &["PipeReply:linked", "Exit", "JobCancel"]); + in_order(&events, &["JobCancel", "JobEnd"]); + in_order(&events, &["JobCancel", "Death"]); +} + +/// Start an owner with a backend B for its jobs. +fn start_with_backend() -> (ActorRef, ActorRef) { + let backend = Backend::default().start(); + let actor = Owner { + backend: Some(backend.clone()), + ..Owner::default() + } + .start(); + lifecycle_trace::watch_deferred(id(&actor)); + (actor, backend) +} + +#[tokio::test] +async fn downstream_request_deferred_then_answered() { + let (actor, backend) = start_with_backend(); + let held = actor + .send_request( + Piped { + downstream: Some(true), + ..piped(1) + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + while backend.request(Held).await.unwrap() == 0 { + sleep(Duration::from_millis(2)).await; + } + backend.send(Release).unwrap(); + assert!(matches!(held.await, Ok(8))); + let events = stop("downstream_later", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:stateB", + "BReply:ok", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +#[tokio::test] +async fn downstream_request_answered_in_its_turn() { + let (actor, _backend) = start_with_backend(); + let got = actor + .request(Piped { + downstream: Some(false), + ..piped(1) + }) + .await; + assert!(matches!(got, Ok(4)), "{got:?}"); + let events = stop("downstream_now", actor).await; + assert_eq!( + events, + [ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:done", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death" + ] + ); +} + +/// A supervised child that traps exits (so its parent's shutdown runs +/// `stopped()`) and whose `stopped()` outlasts two shutdown timeouts. +struct Slow; +impl Actor for Slow { + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } + async fn stopped(&mut self, _: &Context) { + sleep(Duration::from_millis(600)).await; + } +} +impl Handler for Slow { + async fn handle(&mut self, msg: Echo, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(msg.0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cleanup_deadline_before_death_reports_stuck() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("reply-trace-stuck", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let child = tree + .add( + ChildSpec::new("slow", &group, |_| async { Ok::<_, String>(Slow) }) + .shutdown_timeout(Duration::from_millis(100)), + ) + .unwrap(); + let _supervisor = tree.start().await.unwrap(); + let actor = child.current().unwrap(); + let owner: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(owner); + assert!(matches!(actor.request(Echo(2)).await, Ok(2))); + let report = system.shutdown(Duration::from_secs(5)).await; + assert!(report.complete); + let events = write("stuck", owner); + assert_eq!( + events, + ["Send", "Dispatch", "Discharge:ok", "Exit", "Stuck", "Death"] + ); +} + +/// Threads mode: a blocking linked job asks a backend B, which answers in +/// its turn; the job's result answers the caller. +mod threads_mode { + use super::{in_order, trace_dir}; + use spawned_concurrency::{ + lifecycle_trace, + message::Message, + threads::{ + Actor, ActorRef, ActorStart, Context, Handler, JobOptions, JobOutcome, PendingReply, + ReplyTo, + }, + }; + + struct Owner(ActorRef); + impl Actor for Owner { + const MAX_OUTSTANDING: usize = super::BOUND; + } + struct Backend; + impl Actor for Backend {} + + struct Ask; + impl Message for Ask { + type Result = u32; + } + impl Handler for Backend { + fn handle(&mut self, _: Ask, reply: PendingReply<'_, u32>, _: &Context) { + reply.reply(4); + } + } + + struct Piped; + impl Message for Piped { + type Result = u32; + } + struct JobDone { + res: JobOutcome, + token: ReplyTo, + } + impl Message for JobDone { + type Result = (); + } + impl Handler for Owner { + fn handle(&mut self, _: Piped, reply: PendingReply<'_, u32>, ctx: &Context) { + let Ok(token) = reply.defer() else { return }; + let backend = self.0.clone(); + if let Err(e) = ctx.pipe_reply_blocking( + JobOptions::new(), + token, + move |_| backend.request(Ask).unwrap_or(0), + |res, token| JobDone { res, token }, + ) { + drop(e.into_token()); + } + } + } + impl Handler for Owner { + fn handle(&mut self, msg: JobDone, _: PendingReply<'_, ()>, _: &Context) { + match msg.res { + Ok(v) => { + msg.token.reply(v); + } + Err(_) => drop(msg.token), + } + } + } + + #[test] + fn blocking_job_asks_downstream() { + let backend = Backend.start(); + let actor = Owner(backend).start(); + let owner: u64 = actor.id().to_string().parse().unwrap(); + lifecycle_trace::watch_deferred(owner); + assert!(matches!(actor.request(Piped), Ok(4))); + actor.context().stop(); + actor.join(); + let events = lifecycle_trace::take_replies(owner); + let json = format!( + "{{\"name\":\"replies_threads_downstream\",\"schema\":\"replies-v1\",\"bound\":{},\"events\":[\n{}\n]}}\n", + super::BOUND, + events.iter().map(|e| e.to_json()).collect::>().join(",\n") + ); + let dir = trace_dir(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("replies_threads_downstream.json"), json).unwrap(); + let names: Vec = events + .iter() + .map(|e| { + if e.x.is_empty() { + e.action.to_string() + } else { + format!("{}:{}", e.action, e.x) + } + }) + .collect(); + in_order( + &names, + &[ + "Send", + "Dispatch", + "PipeReply:linked", + "CallB", + "BDispatch:done", + "JobNotify:done", + "Discharge:ok", + "Exit", + "Death", + ], + ); + } +} diff --git a/concurrency/tests/resource_cleanup.rs b/concurrency/tests/resource_cleanup.rs index a483fe618..2fad0fee3 100644 --- a/concurrency/tests/resource_cleanup.rs +++ b/concurrency/tests/resource_cleanup.rs @@ -18,14 +18,26 @@ struct Sink; impl tasks::Actor for Sink {} #[cfg(feature = "async")] impl tasks::Handler for Sink { - async fn handle(&mut self, msg: Probe, _: &tasks::Context) { + async fn handle( + &mut self, + msg: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &tasks::Context, + ) { drop(msg.0); + reply.reply(()); } } impl threads::Actor for Sink {} impl threads::Handler for Sink { - fn handle(&mut self, msg: Probe, _: &threads::Context) { + fn handle( + &mut self, + msg: Probe, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &threads::Context, + ) { drop(msg.0); + reply.reply(()); } } fn released(weak: &Weak<()>) { diff --git a/concurrency/tests/semantics_gaps.rs b/concurrency/tests/semantics_gaps.rs index 0043e87c5..07497a3d1 100644 --- a/concurrency/tests/semantics_gaps.rs +++ b/concurrency/tests/semantics_gaps.rs @@ -106,27 +106,50 @@ mod native { } } impl Handler for Observer { - fn handle(&mut self, watch: Watch, ctx: &Context) { + fn handle( + &mut self, + watch: Watch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); + reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, ping: Ping, _: &Context) { + fn handle( + &mut self, + ping: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.send(Event::Ping(ping.0)); + reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, down: Down, _: &Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.send(Event::Down(down.reason)); + reply.reply(()); } } struct Source; impl Actor for Source {} impl Handler>> for Source { - fn handle(&mut self, burst: Burst>, ctx: &Context) { + fn handle( + &mut self, + burst: Burst>, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -134,6 +157,7 @@ mod native { Some(reason) => ctx.stop_with_reason(reason), None => panic!("source panicked"), } + reply.reply(()); } } @@ -168,8 +192,13 @@ mod native { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - fn handle(&mut self, _: Boom, _: &Context) -> u64 { - panic!("request handler panic") + fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + panic!("request handler panic"); } } @@ -178,7 +207,7 @@ mod native { let actor = Panicker.start(); let result = actor.request_with_timeout(Boom, LIMIT); assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); assert!(matches!(actor.wait_exit(), ExitReason::Panic(_))); @@ -187,15 +216,18 @@ mod native { let recipient: Recipient = erased.recipient(); let result = request(&*recipient, Boom, LIMIT); assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); let raw = Panicker.start(); - let receiver = raw.request_raw(Boom).unwrap(); - assert!(receiver - .recv_timeout(LIMIT) - .is_err_and(|e| { e == std::sync::mpsc::RecvTimeoutError::Disconnected })); + let receiver = raw + .send_request(Boom, spawned_concurrency::Deadline::after(LIMIT)) + .unwrap(); + assert!(matches!( + receiver.recv(), + Err(ActorError::ActorStopped(ExitReason::Panic(_))) + )); } } @@ -233,27 +265,50 @@ mod tasks { } } impl Handler for Observer { - async fn handle(&mut self, watch: Watch, ctx: &Context) { + async fn handle( + &mut self, + watch: Watch, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.trap_exit(true); ctx.link(&watch.0); ctx.monitor(&watch.0); + reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, ping: Ping, _: &Context) { + async fn handle( + &mut self, + ping: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.send(Event::Ping(ping.0)); + reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.send(Event::Down(down.reason)); + reply.reply(()); } } struct Source; impl Actor for Source {} impl Handler>> for Source { - async fn handle(&mut self, burst: Burst>, ctx: &Context) { + async fn handle( + &mut self, + burst: Burst>, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { for i in 0..burst.count { burst.to.send(Ping(i)).unwrap(); } @@ -261,6 +316,7 @@ mod tasks { Some(reason) => ctx.stop_with_reason(reason), None => panic!("source panicked"), } + reply.reply(()); } } @@ -305,8 +361,14 @@ mod tasks { struct Ticker(UnboundedSender); impl Actor for Ticker {} impl Handler for Ticker { - async fn handle(&mut self, _: Tick, _: &Context) { + async fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.0.send(Instant::now()); + reply.reply(()); } } @@ -389,13 +451,26 @@ mod tasks { } } impl Handler for Clocked { - async fn handle(&mut self, stamp: Stamp, _: &Context) { + async fn handle( + &mut self, + stamp: Stamp, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = self.log.send((self.generation, stamp.0)); + reply.reply(()); } } impl Handler for Clocked { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected crash"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected crash"); + } } } @@ -404,15 +479,12 @@ mod tasks { async fn timers_are_not_carried_to_a_supervised_replacement() { let system = ActorSystem::new().unwrap(); let group = system.async_group("timers", 1).unwrap(); - let config = SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(1), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(1)) + .jitter(false), + ); let (tx, mut rx) = unbounded_channel(); let first = Arc::new(AtomicBool::new(true)); let mut tree = system.supervisor(config).unwrap(); @@ -470,8 +542,13 @@ mod tasks { struct Panicker; impl Actor for Panicker {} impl Handler for Panicker { - async fn handle(&mut self, _: Boom, _: &Context) -> u64 { - panic!("request handler panic") + async fn handle( + &mut self, + _: Boom, + _reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + panic!("request handler panic"); } } @@ -488,7 +565,7 @@ mod tasks { for actor in actors { let result = actor.request_with_timeout(Boom, LIMIT).await; assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); let reason = timeout(LIMIT, actor.wait_exit()).await.unwrap(); @@ -499,12 +576,14 @@ mod tasks { let recipient: Recipient = erased.recipient(); let result = request(&*recipient, Boom, LIMIT).await; assert!( - matches!(result, Err(ActorError::ActorStopped)), + matches!(result, Err(ActorError::ActorStopped(ExitReason::Panic(_)))), "{result:?}" ); let raw = Panicker.start(); - let receiver = raw.request_raw(Boom).unwrap(); + let receiver = raw + .send_request(Boom, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); assert!(timeout(LIMIT, receiver).await.unwrap().is_err()); assert!(system.shutdown(LIMIT).await.complete); } @@ -598,9 +677,13 @@ fn check_turns(events: &[Turn], counter: usize, outcomes: &[(usize, Result { + Err(ActorError::ActorStopped(_)) => { assert!(!handled.contains(id), "R1: {id} ran but got ActorStopped") } + // Sent after the death point (0.8: `NoProc`, never admitted). + Err(ActorError::NoProc) => { + assert!(!handled.contains(id), "R1: {id} ran but got NoProc") + } Err(other) => panic!("R1: unexpected {other:?} for {id}"), } } @@ -635,21 +718,45 @@ mod native_contract { } } impl Handler for Holder { - fn handle(&mut self, hold: Hold, _: &Context) { - self.seen.lock().unwrap().push("hold"); - hold.entered.send(()).unwrap(); - let _ = hold.release.recv_timeout(LIMIT); + fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.seen.lock().unwrap().push("hold"); + hold.entered.send(()).unwrap(); + let _ = hold.release.recv_timeout(LIMIT); + }; + __reply.reply(__value); } } impl Handler for Holder { - fn handle(&mut self, _: Queued, _: &Context) { - self.seen.lock().unwrap().push("queued"); + fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.seen.lock().unwrap().push("queued"); + }; + __reply.reply(__value); } } impl Handler for Holder { - fn handle(&mut self, _: Asked, _: &Context) -> u64 { - self.seen.lock().unwrap().push("asked"); - 1 + fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = { + self.seen.lock().unwrap().push("asked"); + 1 + }; + __reply.reply(__value); } } @@ -682,7 +789,10 @@ mod native_contract { actor.send(Queued(Payload(drops.clone()))).expect("D3"); replies.push( actor - .request_raw(Asked(Payload(drops.clone()))) + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .expect("D3"), ); } @@ -693,10 +803,9 @@ mod native_contract { WINDOW + 1, "D4/L7: a queued payload outlived join()" ); - for reply in replies { - assert_eq!( - reply.try_recv(), - Err(mpsc::TryRecvError::Disconnected), + for mut reply in replies { + assert!( + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7: a queued reply sender outlived join()" ); } @@ -709,7 +818,7 @@ mod native_contract { assert_eq!(actor.wait_exit(), ExitReason::Normal); assert!(matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } @@ -737,13 +846,21 @@ mod native_contract { } } impl Handler for Turns { - fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.push(Turn::Begin(id)); - let seen = self.counter; - std::thread::yield_now(); - self.counter = seen + 1; - self.push(Turn::End(id)); - id + fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = { + self.push(Turn::Begin(id)); + let seen = self.counter; + std::thread::yield_now(); + self.counter = seen + 1; + self.push(Turn::End(id)); + id + }; + __reply.reply(__value); } } @@ -797,19 +914,41 @@ mod native_contract { struct Backlog(Vec); impl Actor for Backlog {} impl Handler for Backlog { - fn handle(&mut self, hold: Hold, _: &Context) { - hold.entered.send(()).unwrap(); - let _ = hold.release.recv_timeout(LIMIT * 6); + fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + hold.entered.send(()).unwrap(); + let _ = hold.release.recv_timeout(LIMIT * 6); + }; + __reply.reply(__value); } } impl Handler for Backlog { - fn handle(&mut self, Seq(i): Seq, _: &Context) { - self.0.push(i); + fn handle( + &mut self, + Seq(i): Seq, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = { + self.0.push(i); + }; + __reply.reply(__value); } } impl Handler for Backlog { - fn handle(&mut self, _: Count, _: &Context) -> Vec { - std::mem::take(&mut self.0) + fn handle( + &mut self, + _: Count, + __reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + let __value: Vec = std::mem::take(&mut self.0); + __reply.reply(__value); } } let actor = Backlog(Vec::new()).start(); @@ -847,12 +986,20 @@ mod native_contract { } impl Actor for Looper {} impl Handler for Looper { - fn handle(&mut self, _: Loop, ctx: &Context) { - self.count += 1; - let _ = self.reports.send(self.count); - if self.count < 100 { - ctx.send(Loop).unwrap(); - } + fn handle( + &mut self, + _: Loop, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = { + self.count += 1; + let _ = self.reports.send(self.count); + if self.count < 100 { + ctx.send(Loop).unwrap(); + } + }; + __reply.reply(__value); } } @@ -973,21 +1120,48 @@ mod tasks_contract { } } impl Handler for Holder { - async fn handle(&mut self, hold: Hold, _: &Context) { - self.seen.lock().unwrap().push("hold"); - hold.entered.send(()).unwrap(); - let _ = timeout(LIMIT, hold.release).await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.seen.lock().unwrap().push("hold"); + hold.entered.send(()).unwrap(); + let _ = timeout(LIMIT, hold.release).await; + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Queued, _: &Context) { - self.seen.lock().unwrap().push("queued"); + async fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.seen.lock().unwrap().push("queued"); + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Asked, _: &Context) -> u64 { - self.seen.lock().unwrap().push("asked"); - 1 + async fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = async { + self.seen.lock().unwrap().push("asked"); + 1 + } + .await; + __reply.reply(__value); } } @@ -1022,7 +1196,10 @@ mod tasks_contract { actor.send(Queued(Payload(drops.clone()))).expect("D3"); replies.push( actor - .request_raw(Asked(Payload(drops.clone()))) + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .expect("D3"), ); } @@ -1035,7 +1212,7 @@ mod tasks_contract { ); for mut reply in replies { assert!( - matches!(reply.try_recv(), Err(oneshot::error::TryRecvError::Closed)), + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7 {place:?}: a queued reply sender outlived join()" ); } @@ -1048,7 +1225,7 @@ mod tasks_contract { assert_eq!(actor.wait_exit().await, ExitReason::Normal, "{place:?}"); assert!(matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); } placement.finish().await; @@ -1080,14 +1257,23 @@ mod tasks_contract { impl Handler for Turns { /// A read-modify-write that spans an await: it loses updates unless /// the handler keeps the turn while it awaits (M1). - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.push(Turn::Begin(id)); - let seen = self.counter; - spawned_rt::tasks::yield_now().await; - spawned_rt::tasks::yield_now().await; - self.counter = seen + 1; - self.push(Turn::End(id)); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.push(Turn::Begin(id)); + let seen = self.counter; + spawned_rt::tasks::yield_now().await; + spawned_rt::tasks::yield_now().await; + self.counter = seen + 1; + self.push(Turn::End(id)); + id + } + .await; + __reply.reply(__value); } } @@ -1149,19 +1335,43 @@ mod tasks_contract { struct Backlog(Vec); impl Actor for Backlog {} impl Handler for Backlog { - async fn handle(&mut self, hold: Hold, _: &Context) { - hold.entered.send(()).unwrap(); - let _ = timeout(LIMIT * 6, hold.release).await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + hold.entered.send(()).unwrap(); + let _ = timeout(LIMIT * 6, hold.release).await; + } + .await; + __reply.reply(__value); } } impl Handler for Backlog { - async fn handle(&mut self, Seq(i): Seq, _: &Context) { - self.0.push(i); + async fn handle( + &mut self, + Seq(i): Seq, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.0.push(i); + } + .await; + __reply.reply(__value); } } impl Handler for Backlog { - async fn handle(&mut self, _: Count, _: &Context) -> Vec { - std::mem::take(&mut self.0) + async fn handle( + &mut self, + _: Count, + __reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + let __value: Vec = async { std::mem::take(&mut self.0) }.await; + __reply.reply(__value); } } @@ -1212,12 +1422,21 @@ mod tasks_contract { } impl Actor for Looper {} impl Handler for Looper { - async fn handle(&mut self, _: Loop, ctx: &Context) { - self.count += 1; - let _ = self.reports.send(self.count); - if self.count < 100 { - ctx.send(Loop).unwrap(); + async fn handle( + &mut self, + _: Loop, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + self.count += 1; + let _ = self.reports.send(self.count); + if self.count < 100 { + ctx.send(Loop).unwrap(); + } } + .await; + __reply.reply(__value); } } @@ -1296,12 +1515,21 @@ mod tasks_contract { } } impl Handler for Victim { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.handled.fetch_add(1, Ordering::SeqCst); - if self.fault == Fault::Handler { - panic!("victim handler"); + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.handled.fetch_add(1, Ordering::SeqCst); + if self.fault == Fault::Handler { + panic!("victim handler"); + } + id } - id + .await; + __reply.reply(__value); } } @@ -1328,9 +1556,17 @@ mod tasks_contract { if fault == Fault::Stopped { assert!(matches!(first, Ok(1)), "{place:?}"); actor.child_handle().stop(); + } else if fault == Fault::Started { + // Queued before the death point: ActorStopped; admitted + // after it (started() already panicked, as on one CPU): + // NoProc. + assert!( + matches!(first, Err(ActorError::ActorStopped(_) | ActorError::NoProc)), + "{place:?} {fault:?}: {first:?}" + ); } else { assert!( - matches!(first, Err(ActorError::ActorStopped)), + matches!(first, Err(ActorError::ActorStopped(_))), "{place:?} {fault:?}: {first:?}" ); } @@ -1373,9 +1609,18 @@ mod tasks_contract { } } impl Handler for Gate { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - self.handled.fetch_add(1, Ordering::SeqCst); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.handled.fetch_add(1, Ordering::SeqCst); + id + } + .await; + __reply.reply(__value); } } @@ -1395,7 +1640,11 @@ mod tasks_contract { place, ); let replies: Vec<_> = (0..4) - .map(|i| actor.request_raw(Work(i)).unwrap()) + .map(|i| { + actor + .send_request(Work(i), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) .collect(); tokio::task::yield_now().await; assert!( @@ -1438,17 +1687,35 @@ mod tasks_contract { struct Timed; impl Actor for Timed {} impl Handler for Timed { - async fn handle(&mut self, _: Late, _: &Context) { - panic!("T1: a timer fired after its actor stopped"); + async fn handle( + &mut self, + _: Late, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + panic!("T1: a timer fired after its actor stopped"); + } + .await; + __reply.reply(__value); } } impl Handler for Timed { - async fn handle(&mut self, Arm(slot): Arm, ctx: &Context) { - spawned_concurrency::tasks::send_after( - Duration::from_secs(3600), - ctx.clone(), - Late(WhereDropped(slot)), - ); + async fn handle( + &mut self, + Arm(slot): Arm, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + spawned_concurrency::tasks::send_after( + Duration::from_secs(3600), + ctx.clone(), + Late(WhereDropped(slot)), + ); + } + .await; + __reply.reply(__value); } } @@ -1494,8 +1761,17 @@ mod tasks_contract { type Result = (); } impl Handler for Napper { - async fn handle(&mut self, Nap(d): Nap, _: &Context) { - tokio::time::sleep(d).await; + async fn handle( + &mut self, + Nap(d): Nap, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + tokio::time::sleep(d).await; + } + .await; + __reply.reply(__value); } } @@ -1543,18 +1819,27 @@ mod tasks_contract { struct Spawner; impl Actor for Spawner {} impl Handler for Spawner { - async fn handle(&mut self, spawn: Spawn, _: &Context) { - let probe = Payload(spawn.task_drops); - drop(tokio::spawn(async move { - let _probe = probe; - std::future::pending::<()>().await; - })); - let release = spawn.thread_release; - let done = spawn.thread_done; - drop(std::thread::spawn(move || { - let _ = release.recv(); - done.fetch_add(1, Ordering::SeqCst); - })); + async fn handle( + &mut self, + spawn: Spawn, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + let probe = Payload(spawn.task_drops); + drop(tokio::spawn(async move { + let _probe = probe; + std::future::pending::<()>().await; + })); + let release = spawn.thread_release; + let done = spawn.thread_done; + drop(std::thread::spawn(move || { + let _ = release.recv(); + done.fetch_add(1, Ordering::SeqCst); + })); + } + .await; + __reply.reply(__value); } } let system = ActorSystem::new().unwrap(); @@ -1613,18 +1898,16 @@ mod supervision_contract { }; fn config(strategy: SupervisorStrategy, initial: Duration) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 20, - backoff: Backoff { - initial, - maximum: Duration::from_secs(10), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20) + .backoff( + Backoff::default() + .initial(initial) + .maximum(Duration::from_secs(10)) + .jitter(false) + .seed(7), + ) } struct Hold { @@ -1659,33 +1942,78 @@ mod supervision_contract { } impl Actor for Slot {} impl Handler for Slot { - async fn handle(&mut self, hold: Hold, _: &Context) { - let _ = hold.entered.send(()); - let _ = hold.release.await; - panic!("crash with a queue behind it"); + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + let _ = hold.entered.send(()); + let _ = hold.release.await; + panic!("crash with a queue behind it"); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Incr, _: &Context) { - self.n += 1; - self.handled.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Incr, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.n += 1; + self.handled.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.n += 1; - self.n + async fn handle( + &mut self, + _: Next, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + self.n += 1; + self.n + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected crash"); + async fn handle( + &mut self, + _: Crash, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + panic!("injected crash"); + } + .await; + __reply.reply(__value); } } impl Handler for Slot { - async fn handle(&mut self, _: Slow, _: &Context) { - self.handled.fetch_add(1, Ordering::SeqCst); - sleep(Duration::from_millis(10)).await; + async fn handle( + &mut self, + _: Slow, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.handled.fetch_add(1, Ordering::SeqCst); + sleep(Duration::from_millis(10)).await; + } + .await; + __reply.reply(__value); } } @@ -1743,13 +2071,19 @@ mod supervision_contract { let mut replies = Vec::new(); for _ in 0..5 { old.send(Incr).unwrap(); - replies.push(old.request_raw(Next).unwrap()); + replies.push( + old.send_request(Next, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(), + ); } release.send(()).unwrap(); let new = next_generation(&child, generation).await; for reply in replies { assert!( - timeout(LIMIT, reply).await.unwrap().is_err(), + timeout(LIMIT, std::future::IntoFuture::into_future(reply)) + .await + .unwrap() + .is_err(), "V4: a queued request was answered" ); } @@ -1759,7 +2093,7 @@ mod supervision_contract { "V4: state or queue replayed" ); assert_eq!(handled.load(Ordering::SeqCst), 0, "V4: a queued Incr ran"); - assert!(matches!(old.send(Incr), Err(ActorError::ActorStopped))); + assert!(matches!(old.send(Incr), Err(ActorError::NoProc))); assert_eq!(calls.load(Ordering::SeqCst), 2); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(system.shutdown(LIMIT).await.complete); @@ -1922,29 +2256,56 @@ mod supervision_contract { } impl Actor for Late {} impl Handler for Late { - async fn handle(&mut self, _: Fired, _: &Context) { - self.fired.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Fired, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.fired.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Late { - async fn handle(&mut self, _: Down, _: &Context) { - self.downs.fetch_add(1, Ordering::SeqCst); + async fn handle( + &mut self, + _: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + self.downs.fetch_add(1, Ordering::SeqCst); + } + .await; + __reply.reply(__value); } } impl Handler for Late { /// Called while system shutdown is in progress but this actor is /// still running: timers, monitors and registrations are inert. - async fn handle(&mut self, probe: Probe, ctx: &Context) -> (bool, bool) { - send_after(Duration::from_millis(1), ctx.clone(), Fired); - ctx.monitor(&probe.target); - let registered = ctx.register_name("semantics-gaps-late-name").is_ok(); - let started = Late { - fired: self.fired.clone(), - downs: self.downs.clone(), + async fn handle( + &mut self, + probe: Probe, + __reply: spawned_concurrency::PendingReply<'_, (bool, bool)>, + ctx: &Context, + ) { + let __value: (bool, bool) = async { + send_after(Duration::from_millis(1), ctx.clone(), Fired); + ctx.monitor(&probe.target); + let registered = ctx.register_name("semantics-gaps-late-name").is_ok(); + let started = Late { + fired: self.fired.clone(), + downs: self.downs.clone(), + } + .start_on(&ctx.execution_group().unwrap()) + .is_ok(); + (registered, started) } - .start_on(&ctx.execution_group().unwrap()) - .is_ok(); - (registered, started) + .await; + __reply.reply(__value); } } /// Stops first (admitted last); probes `late` from its `stopped()` and @@ -1955,6 +2316,11 @@ mod supervision_contract { result: Arc>>, } impl Actor for Prober { + // 0.8 (E3): the system's shutdown is its parent's exit signal; a + // trapping actor ends through `stopped()`, where the probe runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, ctx: &Context) { let probe = Probe { target: ctx.actor_ref().child_handle(), @@ -2012,6 +2378,10 @@ mod supervision_contract { release: Option>, } impl Actor for Gated { + // 0.8 (E3): shutdown reaches a trapping actor through `stopped()`. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(stopping) = self.stopping.take() { let _ = stopping.send(()); @@ -2023,7 +2393,8 @@ mod supervision_contract { } /// S3, S2: dropping an `ActorSystem` does not block, stops the latest - /// admitted actor first, and signals earlier ones only at the 5 s grace. + /// admitted actor first, and signals earlier ones only at the 5 s grace + /// (with the parent's kill, E3). #[spawned_macros::sim_test] async fn dropping_a_system_shuts_down_in_order_with_a_five_second_grace() { let system = ActorSystem::new().unwrap(); @@ -2059,17 +2430,24 @@ mod supervision_contract { "S2/S3: earlier actor signalled before the later one finished or the grace" ); } - assert_eq!( - timeout(Duration::from_secs(3), earlier.wait_exit()) - .await - .expect("S3: earlier actor not signalled at the 5 s grace"), - ExitReason::Shutdown + let earlier_reason = timeout(Duration::from_secs(3), earlier.wait_exit()) + .await + .expect("S3: earlier actor not signalled at the 5 s grace"); + // 0.8 (E3): at the grace the earlier actor gets the parent's + // `Shutdown`, then every remaining actor the parent's kill; a + // trapping actor that finishes its parent exit first ends `Shutdown`. + assert!( + matches!(earlier_reason, ExitReason::Shutdown | ExitReason::Killed), + "{earlier_reason:?}" ); assert!(later.child_handle().is_alive()); release.send(()).unwrap(); - assert_eq!( - timeout(LIMIT, later.wait_exit()).await.unwrap(), - ExitReason::Shutdown + // `later` is inside `stopped()`: it finishes; a kill that reached it + // before its death point supersedes its reason (X4). + let later_reason = timeout(LIMIT, later.wait_exit()).await.unwrap(); + assert!( + matches!(later_reason, ExitReason::Shutdown | ExitReason::Killed), + "{later_reason:?}" ); } } diff --git a/concurrency/tests/shutdown_publication.rs b/concurrency/tests/shutdown_publication.rs index 041c7eb8f..1aaf4c402 100644 --- a/concurrency/tests/shutdown_publication.rs +++ b/concurrency/tests/shutdown_publication.rs @@ -66,7 +66,12 @@ impl Actor for Worker { } } impl Handler for Worker { - async fn handle(&mut self, msg: Command, _: &Context) { + async fn handle( + &mut self, + msg: Command, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { match msg { Command::Hold { entered, release } => { let _ = entered.send(()); @@ -74,6 +79,7 @@ impl Handler for Worker { } Command::Work => panic!("queued work must be discarded after stop"), } + reply.reply(()); } } struct Cleanup(ChildHandle); @@ -100,10 +106,13 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let (entered, entered_rx) = tokio::sync::oneshot::channel(); let (release, gate) = tokio::sync::oneshot::channel(); let _hold_reply = actor - .request_raw(Command::Hold { - entered, - release: gate, - }) + .send_request( + Command::Hold { + entered, + release: gate, + }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) .unwrap(); tokio::time::timeout(LIMIT, entered_rx) .await @@ -114,7 +123,8 @@ fn admitted_request_racing_shutdown_drops_its_reply_before_join_completes() { let producer = std::thread::spawn(move || { PRODUCER.with(|p| p.set(true)); for _ in 0..1024 { - let reply = sender.request_raw(Command::Work); + let reply = sender + .send_request(Command::Work, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT); if PAUSED.load(Ordering::SeqCst) { return reply; } diff --git a/concurrency/tests/sim.rs b/concurrency/tests/sim.rs index 8f77566f4..be60fab41 100644 --- a/concurrency/tests/sim.rs +++ b/concurrency/tests/sim.rs @@ -39,19 +39,17 @@ fn push(log: &Log, event: String) { } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 3, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - // Fixed jitter seed: otherwise it comes from SystemTime. - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(3) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + // Fixed jitter seed: otherwise it comes from SystemTime. + .seed(7), + ) } struct Crash; @@ -59,25 +57,30 @@ impl Message for Crash { type Result = (); } -/// Records started/stopped. With `crash_after: Some(k)`, `started()` spawns a -/// task that yields `k` times and then sends the actor a panicking message, -/// so it dies a few scheduling steps after it publishes readiness. +/// Records started/stopped. With `fail_start_after: Some(k)`, `started()` +/// yields `k` times and then panics, so the child's first start fails and +/// the supervisor's initial start aborts (S1 in +/// docs/design/otp-exit-signals.md; a crash after readiness would instead be +/// processed after the initial start completes). With `trap`, the actor traps +/// exits, so the system's or supervisor's shutdown (the parent's exit signal) +/// still runs `stopped()` (E3). struct Node { id: u32, - crash_after: Option, + fail_start_after: Option, + trap: bool, log: Log, } impl Actor for Node { async fn started(&mut self, ctx: &Context) { push(&self.log, format!("started:{}", self.id)); - if let Some(yields) = self.crash_after { - let actor = ctx.actor_ref(); - drop(spawned_rt::tasks::spawn(async move { - for _ in 0..yields { - sim::yield_now().await; - } - let _ = actor.send(Crash); - })); + if self.trap { + ctx.trap_exit(true); + } + if let Some(yields) = self.fail_start_after { + for _ in 0..yields { + sim::yield_now().await; + } + panic!("injected first-start failure"); } } async fn stopped(&mut self, _: &Context) { @@ -85,7 +88,12 @@ impl Actor for Node { } } impl Handler for Node { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected crash after readiness"); } } @@ -127,7 +135,8 @@ async fn shutdown_order(log: Log) -> Vec { let group = system.async_group("sim-order", 2).unwrap(); let start = |id| Node { id, - crash_after: None, + fail_start_after: None, + trap: true, log: log.clone(), }; for id in 0..2 { @@ -142,7 +151,8 @@ async fn shutdown_order(log: Log) -> Vec { async move { Ok::<_, String>(Node { id: 100, - crash_after: None, + fail_start_after: None, + trap: true, log, }) } @@ -219,7 +229,8 @@ async fn initial_failure(log: Log, yields: u32) -> Placement { async move { Ok::<_, String>(Node { id: index, - crash_after: (index == 0).then_some(yields), + fail_start_after: (index == 0).then_some(yields), + trap: false, log, }) } @@ -359,7 +370,8 @@ fn blocking_backend_actor_fails_the_run() { let log = Log::default(); let node = Node { id: 1, - crash_after: None, + fail_start_after: None, + trap: false, log, } .start_with_backend(spawned_concurrency::tasks::Backend::Blocking); @@ -385,7 +397,9 @@ fn thread_groups_are_rejected() { /// PR #204 finding 2, pinned: before the v0.7.1 lifecycle fixes (PR #209), /// seed 12 left the launched but never-started child `c1` Failed and seed 2 /// left it Stopped. With PR #209 a failed initial placement finishes every -/// never-started child as Failed, so both seeds must now agree. +/// never-started child as Failed, so both seeds must now agree. (Since S1 the +/// injected failure is `c0`'s first start, so the seeds select different +/// schedules than in v0.7; the expected states are unchanged.) #[test] fn pr204_never_started_child_state_pinned_seeds() { for (seed, expected) in [(12, "Failed"), (2, "Failed")] { @@ -424,26 +438,28 @@ async fn restart_window(out: Arc>>) { let system = ActorSystem::new().unwrap(); let group = system.async_group("sim-window", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForOne, - max_restarts: 3, - restart_window: Duration::from_secs(60), - backoff: Backoff { - initial: Duration::from_secs(1), - maximum: Duration::from_secs(8), - stable_after: Duration::from_secs(3600), - jitter: true, - // Left to spawned_rt::time::entropy: seeded by the simulation. - seed: None, - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForOne) + .max_restarts(3) + .restart_window(Duration::from_secs(60)) + // No fixed seed: jitter is left to spawned_rt::time::entropy, + // seeded by the simulation. + .backoff( + Backoff::default() + .initial(Duration::from_secs(1)) + .maximum(Duration::from_secs(8)) + .stable_after(Duration::from_secs(3600)) + .jitter(true), + ), + ) .unwrap(); let child: SupervisedChild = tree .add(ChildSpec::new("flappy", &group, |_| async { Ok::<_, String>(Node { id: 0, - crash_after: None, + fail_start_after: None, + trap: false, log: Log::default(), }) })) @@ -955,3 +971,119 @@ fn shrink_stays_within_its_run_budget() { assert!(runs <= budget, "budget {budget}: spent {runs}"); } } + +// --------------------------------------------------------------------------- +// Workload: deferred replies parked in a `PendingReplies` and drained at +// once (#261). The drain order decides which caller is answered first, so +// it must not depend on a per-process hash seed. +// --------------------------------------------------------------------------- + +struct Park(u32); +impl Message for Park { + type Result = u32; +} +struct Release; +impl Message for Release { + type Result = Vec; +} +struct Parked; +impl Message for Parked { + type Result = usize; +} + +struct Parker { + pending: spawned_concurrency::tasks::PendingReplies, +} +impl Actor for Parker {} +impl Handler for Parker { + async fn handle( + &mut self, + Park(id): Park, + reply: spawned_concurrency::reply::PendingReply<'_, u32>, + _: &Context, + ) { + if let Ok(token) = reply.defer() { + assert!(self.pending.insert(id, token).is_ok()); + } + } +} +impl Handler for Parker { + async fn handle( + &mut self, + _: Parked, + reply: spawned_concurrency::reply::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.pending.len()); + } +} +impl Handler for Parker { + async fn handle( + &mut self, + _: Release, + reply: spawned_concurrency::reply::PendingReply<'_, Vec>, + _: &Context, + ) { + let mut order = Vec::new(); + for (id, token) in self.pending.drain() { + token.reply(id); + order.push(id); + } + reply.reply(order); + } +} + +/// Sixteen callers park in an order the scheduler picks; one `Release` +/// drains them. Returns the drain order; callers log their answers. +async fn drain_pending(log: Log) -> Vec { + const CALLERS: u32 = 16; + let parker = Parker { + pending: spawned_concurrency::tasks::PendingReplies::with_capacity(CALLERS as usize), + } + .start(); + let callers: Vec<_> = (0..CALLERS) + .map(|id| { + let (parker, log) = (parker.clone(), log.clone()); + spawned_rt::tasks::spawn(async move { + let answer = parker.request(Park(id)).await.unwrap(); + push(&log, format!("answered:{answer}")); + }) + }) + .collect(); + while parker.request(Parked).await.unwrap() < CALLERS as usize { + sim::yield_now().await; + } + let order = parker.request(Release).await.unwrap(); + for caller in callers { + caller.await.unwrap(); + } + parker.context().stop(); + parker.join().await; + order +} + +/// #261: the same seed gives the same drain order, caller log, scheduler +/// trace and tape. With a `HashMap` table each run drew its own +/// `RandomState`, so the order (and with it the schedule) differed. +#[test] +fn pending_replies_drain_is_reproducible() { + let record = || { + let rt = Runtime::with_seed(7); + rt.record_trace(); + rt.record_tape(); + let log: Log = Arc::default(); + let order = rt.block_on(drain_pending(log.clone())); + let log = log.lock().unwrap_or_else(|p| p.into_inner()).clone(); + (order, log, rt.trace(), rt.tape()) + }; + let (order, log, trace, tape) = record(); + let again = record(); + assert_eq!(order.len(), 16); + assert_eq!(again.0, order, "drain order differs between same-seed runs"); + assert_eq!(again.1, log, "caller answers differ between same-seed runs"); + assert_eq!( + again.2, trace, + "scheduler trace differs between same-seed runs" + ); + assert_eq!(again.3, tape, "tape differs between same-seed runs"); +} diff --git a/concurrency/tests/sim_explore.rs b/concurrency/tests/sim_explore.rs index 7f060e6ac..b1f9080d4 100644 --- a/concurrency/tests/sim_explore.rs +++ b/concurrency/tests/sim_explore.rs @@ -56,20 +56,39 @@ impl Message for Log { type Result = Vec; } impl Handler for Store { - async fn handle(&mut self, _: Get, _: &Context) -> u64 { - self.log.push(Access::Read(self.value)); - self.value + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.log.push(Access::Read(self.value)); + self.value + }; + reply.reply(__reply_value); } } impl Handler for Store { - async fn handle(&mut self, Set(value): Set, _: &Context) { + async fn handle( + &mut self, + Set(value): Set, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.log.push(Access::Write(value)); self.value = value; + reply.reply(()); } } impl Handler for Store { - async fn handle(&mut self, _: Log, _: &Context) -> Vec { - self.log.clone() + async fn handle( + &mut self, + _: Log, + reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + reply.reply(self.log.clone()); } } @@ -87,12 +106,18 @@ impl Message for Bump { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { for _ in 0..self.noise { spawned_rt::tasks::yield_now().await; } let value = self.store.request(Get).await.unwrap(); self.store.request(Set(value + 1)).await.unwrap(); + reply.reply(()); } } @@ -116,7 +141,10 @@ async fn bump_all(noises: &[u32]) -> Vec { .collect(); let replies: Vec<_> = workers .iter() - .map(|w| w.request_raw(Bump).unwrap()) + .map(|w| { + w.send_request(Bump, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap() + }) .collect(); for reply in replies { reply.await.unwrap(); @@ -180,55 +208,46 @@ async fn lost_update_three_workers() { // terminal state depends on which of two finishes runs last. // --------------------------------------------------------------------------- -struct Crash; -impl Message for Crash { - type Result = (); -} struct Node { - crash: bool, + fail_start: bool, } -/// As in `tests/sim.rs` with zero yields: `started()` spawns a task that sends -/// the actor a panicking message. +/// As in `tests/sim.rs` with zero yields: `started()` panics, so the first +/// child's first start fails and the initial start aborts (S1 in +/// docs/design/otp-exit-signals.md). impl Actor for Node { - async fn started(&mut self, ctx: &Context) { - if self.crash { - let actor = ctx.actor_ref(); - drop(spawned_rt::tasks::spawn(async move { - let _ = actor.send(Crash); - })); + async fn started(&mut self, _: &Context) { + if self.fail_start { + panic!("injected first-start failure"); } } } -impl Handler for Node { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected crash after readiness"); - } -} /// States (`Debug` of the state, generation) of three OneForAll children -/// whose first child crashes right after `started()`. +/// whose first child fails its first start. async fn initial_failure() -> Vec<(String, u64)> { let system = ActorSystem::new().unwrap(); let group = system.async_group("explore-initial", 2).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 3, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(3) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false) + .seed(7), + ), + ) .unwrap(); let children: Vec<_> = (0..3) .map(|index| { tree.add( ChildSpec::new(format!("c{index}"), &group, move |_| async move { - Ok::<_, String>(Node { crash: index == 0 }) + Ok::<_, String>(Node { + fail_start: index == 0, + }) }) .restart(RestartMode::Permanent), ) diff --git a/concurrency/tests/sim_ordering.rs b/concurrency/tests/sim_ordering.rs index 1eea34695..39161f915 100644 --- a/concurrency/tests/sim_ordering.rs +++ b/concurrency/tests/sim_ordering.rs @@ -28,7 +28,6 @@ use std::sync::{ Arc, Mutex, }; use std::time::Duration; -use tokio::sync::oneshot::error::TryRecvError; const LIMIT: Duration = Duration::from_secs(5); @@ -70,17 +69,35 @@ impl Message for Echo { type Result = (usize, usize); } impl Handler for Sink { - async fn handle(&mut self, Put(sender, seq): Put, _: &Context) { - push(&self.0, (sender, seq)); + async fn handle( + &mut self, + Put(sender, seq): Put, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, (sender, seq)); + } + .await; + __reply.reply(__value); } } impl Handler for Sink { - async fn handle(&mut self, Echo(sender, seq): Echo, _: &Context) -> (usize, usize) { - push(&self.0, (sender, seq)); - // Hold the turn across an await so a reply could be misrouted only - // if replies were not bound to their own request. - yield_now().await; - (sender, seq) + async fn handle( + &mut self, + Echo(sender, seq): Echo, + __reply: spawned_concurrency::PendingReply<'_, (usize, usize)>, + _: &Context, + ) { + let __value: (usize, usize) = async { + push(&self.0, (sender, seq)); + // Hold the turn across an await so a reply could be misrouted only + // if replies were not bound to their own request. + yield_now().await; + (sender, seq) + } + .await; + __reply.reply(__value); } } @@ -195,11 +212,20 @@ impl Message for Work { type Result = usize; } impl Handler for Turns { - async fn handle(&mut self, Work(id): Work, _: &Context) -> usize { - push(&self.0, Turn::Begin(id)); - yields(2).await; - push(&self.0, Turn::End(id)); - id + async fn handle( + &mut self, + Work(id): Work, + __reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __value: usize = async { + push(&self.0, Turn::Begin(id)); + yields(2).await; + push(&self.0, Turn::End(id)); + id + } + .await; + __reply.reply(__value); } } @@ -243,13 +269,11 @@ async fn turns_body() { // L7: join() returned, so stopped() has completed and new work is refused. let events = snapshot(&events); assert_eq!(events.last(), Some(&Turn::StoppedEnd), "L7: {events:?}"); - assert!(matches!( - actor.send(Work(99)), - Err(ActorError::ActorStopped) - )); + // 0.8: past the death point a send or request is `NoProc`. + assert!(matches!(actor.send(Work(99)), Err(ActorError::NoProc))); assert!(matches!( actor.request_with_timeout(Work(99), LIMIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); // L1/M1: started, then whole non-overlapping turns, then stopped. @@ -284,12 +308,16 @@ async fn turns_body() { assert!(handled.contains(&id), "R1: Ok without a turn for {id}"); } Ok(None) => {} - Err(ActorError::ActorStopped) => { + Err(ActorError::ActorStopped(_)) => { assert!( !handled.contains(&id), "R1: ran but caller got ActorStopped" ); } + // Sent after the death point (0.8): never admitted. + Err(ActorError::NoProc) => { + assert!(!handled.contains(&id), "R1: ran but caller got NoProc"); + } Err(other) => panic!("R1: unexpected outcome {other:?} for {id}"), } } @@ -350,21 +378,48 @@ impl Actor for Holder { } } impl Handler for Holder { - async fn handle(&mut self, hold: Hold, _: &Context) { - push(&self.0, Seen::Hold); - let _ = hold.entered.send(()); - let _ = hold.release.await; + async fn handle( + &mut self, + hold: Hold, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Seen::Hold); + let _ = hold.entered.send(()); + let _ = hold.release.await; + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Queued, _: &Context) { - push(&self.0, Seen::Queued); + async fn handle( + &mut self, + _: Queued, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Seen::Queued); + } + .await; + __reply.reply(__value); } } impl Handler for Holder { - async fn handle(&mut self, _: Asked, _: &Context) -> u64 { - push(&self.0, Seen::Asked); - 1 + async fn handle( + &mut self, + _: Asked, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __value: u64 = async { + push(&self.0, Seen::Asked); + 1 + } + .await; + __reply.reply(__value); } } @@ -398,7 +453,14 @@ async fn stop_discards_body() { // D3: accepted even after the stop request. actor.send(Queued(Payload(drops.clone()))).unwrap(); } else { - replies.push(actor.request_raw(Asked(Payload(drops.clone()))).unwrap()); + replies.push( + actor + .send_request( + Asked(Payload(drops.clone())), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(), + ); } yield_now().await; } @@ -430,7 +492,7 @@ async fn stop_discards_body() { ); for mut reply in replies { assert!( - matches!(reply.try_recv(), Err(TryRecvError::Closed)), + matches!(reply.try_recv(), Some(Err(ActorError::ActorStopped(_)))), "D4/L7: a queued reply sender outlived the exit" ); } @@ -443,7 +505,7 @@ async fn stop_discards_body() { assert!( matches!( actor.send(Queued(Payload(drops.clone()))), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) ), "L7: send after exit" ); @@ -487,10 +549,19 @@ impl Message for Watch { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, Watch(target): Watch, ctx: &Context) { - ctx.trap_exit(true); - ctx.link(&target); - ctx.monitor(&target); + async fn handle( + &mut self, + Watch(target): Watch, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + ctx.trap_exit(true); + ctx.link(&target); + ctx.monitor(&target); + } + .await; + __reply.reply(__value); } } struct Ping(usize); @@ -498,8 +569,17 @@ impl Message for Ping { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, Ping(i): Ping, _: &Context) { - push(&self.0, Notice::Ping(i)); + async fn handle( + &mut self, + Ping(i): Ping, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Ping(i)); + } + .await; + __reply.reply(__value); } } struct Noise; @@ -507,14 +587,32 @@ impl Message for Noise { type Result = (); } impl Handler for Observer { - async fn handle(&mut self, _: Noise, _: &Context) { - push(&self.0, Notice::Noise); + async fn handle( + &mut self, + _: Noise, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Noise); + } + .await; + __reply.reply(__value); } } impl Handler for Observer { - async fn handle(&mut self, down: Down, _: &Context) { - push(&self.0, Notice::Down(down.reason)); - let _ = self.1.send(()); + async fn handle( + &mut self, + down: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + let __value: () = async { + push(&self.0, Notice::Down(down.reason)); + let _ = self.1.send(()); + } + .await; + __reply.reply(__value); } } @@ -539,17 +637,26 @@ impl Message for Burst { type Result = (); } impl Handler for Source { - async fn handle(&mut self, burst: Burst, ctx: &Context) { - let _ = burst.entered.send(()); - for i in 0..PINGS { - burst.to.send(Ping(i)).unwrap(); - yield_now().await; - } - match burst.end { - End::SelfStop => ctx.stop_with_reason(ExitReason::Shutdown), - End::Panic => panic!("source panicked"), - End::External => {} + async fn handle( + &mut self, + burst: Burst, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + let _ = burst.entered.send(()); + for i in 0..PINGS { + burst.to.send(Ping(i)).unwrap(); + yield_now().await; + } + match burst.end { + End::SelfStop => ctx.stop_with_reason(ExitReason::Shutdown), + End::Panic => panic!("source panicked"), + End::External => {} + } } + .await; + __reply.reply(__value); } } diff --git a/concurrency/tests/sim_turn_budget.rs b/concurrency/tests/sim_turn_budget.rs index 7e3f5c757..54a5a5ee5 100644 --- a/concurrency/tests/sim_turn_budget.rs +++ b/concurrency/tests/sim_turn_budget.rs @@ -28,8 +28,17 @@ impl Message for Nap { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _msg: Nap, _ctx: &Context) { - std::thread::sleep(OVER); + async fn handle( + &mut self, + _msg: Nap, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { + let __value: () = async { + std::thread::sleep(OVER); + } + .await; + __reply.reply(__value); } } @@ -39,15 +48,24 @@ impl Message for Spin { type Result = u64; } impl Handler for Worker { - async fn handle(&mut self, _msg: Spin, _ctx: &Context) -> u64 { - // The real clock: `std::time::Instant` is virtual inside a - // simulated poll and does not move while a handler spins. - let start = spawned_rt::tasks::sim::wall_now(); - let mut n = 0u64; - while spawned_rt::tasks::sim::wall_now() - start < OVER { - n = std::hint::black_box(n.wrapping_add(1)); + async fn handle( + &mut self, + _msg: Spin, + __reply: spawned_concurrency::PendingReply<'_, u64>, + _ctx: &Context, + ) { + let __value: u64 = async { + // The real clock: `std::time::Instant` is virtual inside a + // simulated poll and does not move while a handler spins. + let start = spawned_rt::tasks::sim::wall_now(); + let mut n = 0u64; + while spawned_rt::tasks::sim::wall_now() - start < OVER { + n = std::hint::black_box(n.wrapping_add(1)); + } + n } - n + .await; + __reply.reply(__value); } } @@ -58,12 +76,21 @@ impl Message for Chunked { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _msg: Chunked, _ctx: &Context) { - for _ in 0..25 { - std::thread::sleep(Duration::from_millis(20)); - sim::yield_now().await; + async fn handle( + &mut self, + _msg: Chunked, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { + let __value: () = async { + for _ in 0..25 { + std::thread::sleep(Duration::from_millis(20)); + sim::yield_now().await; + } + sim::sleep(Duration::from_secs(1)).await; } - sim::sleep(Duration::from_secs(1)).await; + .await; + __reply.reply(__value); } } diff --git a/concurrency/tests/stack_budget.rs b/concurrency/tests/stack_budget.rs index 0d21e5543..6dd65ac55 100644 --- a/concurrency/tests/stack_budget.rs +++ b/concurrency/tests/stack_budget.rs @@ -120,13 +120,21 @@ mod tasks_chain { } impl Actor for Hop {} impl Handler for Hop { - async fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { - let mut block = msg.block; - stamp(&mut block, self.index); - match &self.next { - Some(next) => next.forward(block).await.expect("downstream hop"), - None => block, - } + async fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::PendingReply<'_, Block>, + _: &Context, + ) { + let __reply_value: Block = { + let mut block = msg.block; + stamp(&mut block, self.index); + match &self.next { + Some(next) => next.forward(block).await.expect("downstream hop"), + None => block, + } + }; + reply.reply(__reply_value); } } @@ -218,15 +226,26 @@ mod tasks_chain { } impl Actor for Flaky {} impl Handler for Flaky { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { if self.generation == 1 { panic!("injected stack-budget crash"); } + reply.reply(()); } } impl Handler for Flaky { - async fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { - msg.block + async fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::PendingReply<'_, Block>, + _: &Context, + ) { + reply.reply(msg.block); } } @@ -238,17 +257,13 @@ mod tasks_chain { system.async_group("stack-async", 2).unwrap(), system.thread_group("stack-thread", 4).unwrap(), ] { - let config = SupervisorConfig { - max_restarts: 10, - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }; + let config = SupervisorConfig::default().max_restarts(10).backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(1), + ); let mut tree = system.supervisor(config).unwrap(); let child = tree .add(ChildSpec::new("flaky", &group, |ctx| { @@ -298,13 +313,21 @@ mod threads_chain { } impl Actor for Hop {} impl Handler for Hop { - fn handle(&mut self, msg: relay::Forward, _: &Context) -> Block { - let mut block = msg.block; - stamp(&mut block, self.index); - match &self.next { - Some(next) => next.forward(block).unwrap(), - None => block, - } + fn handle( + &mut self, + msg: relay::Forward, + reply: spawned_concurrency::PendingReply<'_, Block>, + _: &Context, + ) { + let __reply_value: Block = { + let mut block = msg.block; + stamp(&mut block, self.index); + match &self.next { + Some(next) => next.forward(block).unwrap(), + None => block, + } + }; + reply.reply(__reply_value); } } diff --git a/concurrency/tests/startup_admission.rs b/concurrency/tests/startup_admission.rs index 10d2f7e22..a234ab132 100644 --- a/concurrency/tests/startup_admission.rs +++ b/concurrency/tests/startup_admission.rs @@ -33,8 +33,15 @@ impl Drop for Queued { } } impl spawned_concurrency::tasks::Handler for Probe { - async fn handle(&mut self, _: Queued, _: &Context) { - panic!("unstarted handler ran"); + async fn handle( + &mut self, + _: Queued, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("unstarted handler ran"); + } } } impl Actor for Probe { @@ -82,9 +89,18 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() let actor = Probe(counts.clone()).start_on(&group).unwrap(); let dropped = Arc::new(AtomicUsize::new(0)); let replies: Vec<_> = (0..2) - .map(|_| actor.request_raw(Queued(dropped.clone())).unwrap()) + .map(|_| { + actor + .send_request( + Queued(dropped.clone()), + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap() + }) .collect(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); release.send(()).unwrap(); spawned_rt::tasks::timeout(LIMIT, blocker.join()) .await @@ -105,7 +121,7 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() assert_eq!(counts.dropped.load(Ordering::SeqCst), 1); assert_eq!(dropped.load(Ordering::SeqCst), 2); assert_eq!( - actor.child_handle().mailbox_cleanup_failure().as_deref(), + actor.child_handle().cleanup_failure().as_deref(), Some("unstarted queued destructor") ); for reply in replies { @@ -121,7 +137,9 @@ async fn managed_actor_stopped_before_first_poll_runs_no_hooks_but_drops_state() async fn legacy_actor_stopped_before_first_poll_keeps_both_hooks() { let counts = Arc::new(Counts::default()); let actor = Probe(counts.clone()).start(); // this runtime has not yielded - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!( spawned_rt::tasks::timeout(LIMIT, actor.wait_exit()) .await diff --git a/concurrency/tests/supervision.rs b/concurrency/tests/supervision.rs index 4e308ff37..a7b05baf5 100644 --- a/concurrency/tests/supervision.rs +++ b/concurrency/tests/supervision.rs @@ -20,17 +20,15 @@ use std::{ }; const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(8), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(8)) + .jitter(false) + .seed(7), + ) } struct Next; impl Message for Next { @@ -43,14 +41,29 @@ impl Message for Crash { struct Counter(usize); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.0 += 1; - self.0 + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { + self.0 += 1; + self.0 + }; + reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected child failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected child failure"); + } } } @@ -138,7 +151,9 @@ async fn restart_modes_distinguish_normal_shutdown_and_panic() { let generation = child.snapshot().generation; match &exit { ExitReason::Normal => old.child_handle().stop(), - ExitReason::Shutdown => old.child_handle().shutdown(), + ExitReason::Shutdown => old + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown), _ => assert!(old.request(Crash).await.is_err()), } // The child really exited with the reason under test. @@ -235,9 +250,11 @@ async fn failed_factories_cancel_owned_helpers_before_retry_and_catch_panics() { }) .unwrap(); drop(helper); + // v0.8 (S1): a first start is never retried, so the failed + // constructions are replacements after a crash. match attempt { - 0 => Err("injected construction error".to_string()), - 1 => panic!("injected factory panic"), + 1 => Err("injected construction error".to_string()), + 2 => panic!("injected factory panic"), _ => Ok(Counter(0)), } } @@ -247,9 +264,14 @@ async fn failed_factories_cancel_owned_helpers_before_retry_and_catch_panics() { .await .unwrap() .unwrap(); - assert_eq!(attempts.load(Ordering::SeqCst), 3); - assert_eq!(child.snapshot().generation, 3); - assert_eq!(child.current().unwrap().request(Next).await.unwrap(), 1); + assert!(child.current().unwrap().request(Crash).await.is_err()); + let replacement = spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_eq!(attempts.load(Ordering::SeqCst), 4); + assert_eq!(child.snapshot().generation, 4); + assert_eq!(replacement.request(Next).await.unwrap(), 1); assert!(supervisor.shutdown(LIMIT).await.complete); assert!(!live.load(Ordering::SeqCst)); assert!(system.shutdown(LIMIT).await.complete); @@ -294,8 +316,15 @@ impl Actor for SlowHelper { } } impl Handler for SlowHelper { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected failure with active helper"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected failure with active helper"); + } } } // Not simulated: a helper blocks an OS worker on purpose. @@ -355,14 +384,25 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // Traps exits so stopped() also runs on its parent's shutdown (E3). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { let _ = self.entered.take().unwrap().send(()); let _ = self.release.take().unwrap().await; } } impl Handler for SlowStop { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("failure before stuck stopped hook"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("failure before stuck stopped hook"); + } } } // Not simulated: a helper blocks an OS worker on purpose. @@ -398,7 +438,9 @@ async fn stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publicatio .unwrap(); let supervisor = within!(tree.start()).await.unwrap(); let old = child.current().unwrap(); - let response = old.request_raw(Crash).unwrap(); + let response = old + .send_request(Crash, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); spawned_rt::tasks::timeout(LIMIT, entered) .await .unwrap() @@ -416,11 +458,16 @@ async fn stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publicatio let report = system.shutdown(Duration::from_millis(5)).await; assert!(!report.complete); assert!(report.unfinished_actors.contains(&old.id())); + // Let the system's kill at grace land before releasing stopped(): the + // report can return before the kill reaches the actor under load. + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; release.send(()).unwrap(); let report = report.wait(LIMIT).await; assert!(report.complete); assert_eq!(calls.load(Ordering::SeqCst), 1); - assert!(matches!(old.exit_reason(), Some(ExitReason::Panic(_)))); + // The system's kill at grace reached the actor inside its stuck + // stopped() (before its death point): Killed replaces the panic (E1). + assert_eq!(old.exit_reason(), Some(ExitReason::Killed)); } struct Ordered { @@ -428,7 +475,10 @@ struct Ordered { log: Arc>>, } impl Actor for Ordered { - async fn started(&mut self, _: &Context) { + async fn started(&mut self, ctx: &Context) { + // v0.8: only a trapping child runs stopped() on its parent's + // shutdown (E3, decision 2); the order is observed through it. + ctx.trap_exit(true); self.log .lock() .unwrap() @@ -546,14 +596,29 @@ impl Drop for RaceActor { } impl Actor for RaceActor {} impl Handler for RaceActor { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("racing child failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("racing child failure"); + } } } impl Handler for RaceActor { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.count += 1; - self.count + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { + self.count += 1; + self.count + }; + reply.reply(__reply_value); } } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] @@ -778,18 +843,18 @@ async fn stuck_supervisor_does_not_prevent_earlier_standalone_shutdown_after_sys .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, earlier.wait_exit()) - .await - .unwrap(), - ExitReason::Shutdown - ); - assert_eq!( - spawned_rt::tasks::timeout(LIMIT, inner_actor.wait_exit()) + // Signalled after the system's grace expired, they also get the kill + // at grace (E3): Shutdown if they reached their death point first, + // otherwise Killed. + for actor in [earlier.child_handle(), inner_actor.child_handle()] { + let reason = spawned_rt::tasks::timeout(LIMIT, actor.wait_exit_async()) .await - .unwrap(), - ExitReason::Shutdown - ); + .unwrap(); + assert!( + matches!(reason, ExitReason::Shutdown | ExitReason::Killed), + "{reason:?}" + ); + } release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); } @@ -836,8 +901,15 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected child failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected child failure"); + } } } @@ -994,8 +1066,10 @@ async fn transient_startup_timeout_retries_after_actual_cleanup() { ChildSpec::new("slow-init", &group, move |ctx| { attempts.fetch_add(1, Ordering::SeqCst); async move { + // v0.8 (S1): a first start is never retried, so the + // slow start is the first replacement's. Ok::<_, String>(SlowInit { - first: ctx.generation() == 1, + first: ctx.generation() == 2, }) } }) @@ -1007,8 +1081,13 @@ async fn transient_startup_timeout_retries_after_actual_cleanup() { .await .unwrap() .unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); - assert_eq!(child.snapshot().generation, 2); + child.current().unwrap().child_handle().kill(); + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); + assert_eq!(child.snapshot().generation, 3); assert_eq!(child.snapshot().last_exit, Some(ExitReason::Shutdown)); assert!(child.current().is_ok()); assert!(supervisor.shutdown(LIMIT).await.complete); @@ -1088,17 +1167,16 @@ async fn backoff_exponent_resets_only_after_a_stable_generation() { let system = ActorSystem::new().unwrap(); let group = system.async_group("backoff-reset", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 20, - backoff: Backoff { - initial: Duration::from_millis(20), - maximum: Duration::from_secs(2), - stable_after: Duration::from_millis(300), - jitter: false, - seed: Some(7), - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().max_restarts(20).backoff( + Backoff::default() + .initial(Duration::from_millis(20)) + .maximum(Duration::from_secs(2)) + .stable_after(Duration::from_millis(300)) + .jitter(false) + .seed(7), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("flappy", &group, |_| async { @@ -1264,11 +1342,15 @@ async fn system_deadline_signals_remaining_supervised_siblings() { .await .unwrap() .unwrap(); - assert_eq!( - spawned_rt::tasks::timeout(Duration::from_secs(1), actor.wait_exit()) - .await - .expect("earlier sibling was not stopped after the system deadline"), - ExitReason::Shutdown + let reason = spawned_rt::tasks::timeout(Duration::from_secs(1), actor.wait_exit()) + .await + .expect("earlier sibling was not stopped after the system deadline"); + // Signalled after the system deadline: by the supervisor's `Shutdown` + // or by the grace-expiry kill (E3), whichever reaches it first (on one + // CPU the shutdown can). + assert!( + matches!(reason, ExitReason::Killed | ExitReason::Shutdown), + "{reason:?}" ); release.send(()).unwrap(); assert!(report.wait(LIMIT).await.complete); @@ -1305,8 +1387,15 @@ impl Actor for Named { } } impl Handler for Named { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("crash a registered child"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("crash a registered child"); + } } } @@ -1374,18 +1463,18 @@ async fn one_for_all_restart_waits_for_its_backoff() { let system = ActorSystem::new().unwrap(); let group = system.async_group("ofa-backoff", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::OneForAll, - max_restarts: 5, - backoff: Backoff { - initial: Duration::from_millis(300), - maximum: Duration::from_secs(2), - jitter: false, - seed: Some(1), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(5) + .backoff( + Backoff::default() + .initial(Duration::from_millis(300)) + .maximum(Duration::from_secs(2)) + .jitter(false) + .seed(1), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("a", &group, |_| async { @@ -1420,43 +1509,19 @@ fn supervisor_config_limits_are_validated_at_their_boundaries() { }; let base = SupervisorConfig::default; assert!(accepts(base())); - assert!(rejects(SupervisorConfig { - restart_window: Duration::ZERO, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_children: 0, - ..base() - })); - assert!(accepts(SupervisorConfig { - max_children: 65536, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_children: 65537, - ..base() - })); - assert!(accepts(SupervisorConfig { - max_restarts: 65536, - ..base() - })); - assert!(rejects(SupervisorConfig { - max_restarts: 65537, - ..base() - })); - let backoff = |initial, maximum| Backoff { - initial: Duration::from_millis(initial), - maximum: Duration::from_millis(maximum), - ..Backoff::default() + assert!(rejects(base().restart_window(Duration::ZERO))); + assert!(rejects(base().max_children(0))); + assert!(accepts(base().max_children(65536))); + assert!(rejects(base().max_children(65537))); + assert!(accepts(base().max_restarts(65536))); + assert!(rejects(base().max_restarts(65537))); + let backoff = |initial, maximum| { + Backoff::default() + .initial(Duration::from_millis(initial)) + .maximum(Duration::from_millis(maximum)) }; - assert!(accepts(SupervisorConfig { - backoff: backoff(10, 10), - ..base() - })); - assert!(rejects(SupervisorConfig { - backoff: backoff(11, 10), - ..base() - })); + assert!(accepts(base().backoff(backoff(10, 10)))); + assert!(rejects(base().backoff(backoff(11, 10)))); } /// T4/V5: on native threads (`thread_group`) the restart window runs on the @@ -1477,11 +1542,7 @@ fn native_restart_window_expires_on_the_monotonic_clock() { let system = ActorSystem::new().unwrap(); let group = system.thread_group("window", 4).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - max_restarts: 1, - restart_window: window, - ..config(1) - }) + .supervisor(config(1).max_restarts(1).restart_window(window)) .unwrap(); let child = tree .add(ChildSpec::new("native", &group, |_| async { diff --git a/concurrency/tests/supervision_strategies.rs b/concurrency/tests/supervision_strategies.rs index 714c56bfd..008539f7e 100644 --- a/concurrency/tests/supervision_strategies.rs +++ b/concurrency/tests/supervision_strategies.rs @@ -6,30 +6,30 @@ mod deadline; use spawned_concurrency::{ message::Message, - supervision::{Backoff, ChildSpec, RestartMode, SupervisorConfig, SupervisorStrategy}, + supervision::{ + Backoff, ChildSpec, RestartMode, SupervisorConfig, SupervisorError, SupervisorStrategy, + }, system::ActorSystem, tasks::{Actor, Context, Handler}, }; use std::{ sync::{ - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicUsize, Ordering}, Arc, Mutex, }, time::Duration, }; const LIMIT: Duration = Duration::from_secs(5); fn policy(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 100, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(4), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(100) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(4)) + .jitter(false), + ) } struct Next; impl Message for Next { @@ -73,20 +73,42 @@ impl Drop for Probe { } } impl Handler for Probe { - async fn handle(&mut self, _: Next, _: &Context) -> usize { - self.value += 1; - self.value + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { + self.value += 1; + self.value + }; + reply.reply(__reply_value); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("injected cohort failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected cohort failure"); + } } } impl Handler for Probe { - async fn handle(&mut self, msg: CrashTogether, _: &Context) { - msg.0.wait().await; - panic!("simultaneous cohort failure"); + async fn handle( + &mut self, + msg: CrashTogether, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + msg.0.wait().await; + panic!("simultaneous cohort failure"); + } } } @@ -149,6 +171,7 @@ async fn strategies_select_affected_children_and_reconstruct_in_dependency_order SupervisorStrategy::OneForOne => index == 1, SupervisorStrategy::OneForAll => true, SupervisorStrategy::RestForOne => index >= 1, + other => unreachable!("strategy {other:?} not under test"), }; if affected { let next = @@ -315,49 +338,91 @@ async fn failed_cohort_construction_rebuilds_the_whole_cohort_like_otp() { assert!(system.shutdown(LIMIT).await.complete); } +/// OTP charges one restart per restart action: a OneForAll or RestForOne +/// recovery round costs one grant however many children it rebuilds. With +/// three children and `max_restarts = 2`, two rounds rebuild the cohort and +/// the third failure exhausts the budget (per-child charging would have +/// failed the supervisor while rebuilding the first round). #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn cohort_reconstruction_consumes_shared_factory_attempt_budget() { - let system = ActorSystem::new().unwrap(); - let group = system.async_group("cohort-budget", 1).unwrap(); - let mut config = policy(SupervisorStrategy::OneForAll); - config.max_restarts = 2; - let mut tree = system.supervisor(config).unwrap(); - let mut children = Vec::new(); - let live = Arc::new((0..3).map(|_| AtomicUsize::new(0)).collect::>()); - let events = Arc::new(Mutex::new(Vec::new())); - for index in 0..3 { - let live = live.clone(); - let events = events.clone(); - children.push( - tree.add(ChildSpec::new( - format!("child-{index}"), - &group, - move |ctx| { - let live = live.clone(); - let events = events.clone(); - async move { - assert_eq!(live[index].fetch_add(1, Ordering::SeqCst), 0); - Ok::<_, String>(Probe { - index, - generation: ctx.generation() as usize, - value: 0, - live, - events, - }) - } - }, - )) - .unwrap(), +async fn cohort_reconstruction_charges_one_restart_per_round() { + for strategy in [ + SupervisorStrategy::OneForAll, + SupervisorStrategy::RestForOne, + ] { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("cohort-budget", 1).unwrap(); + let mut config = policy(strategy); + config.max_restarts = 2; + let mut tree = system.supervisor(config).unwrap(); + let mut children = Vec::new(); + let live = Arc::new((0..3).map(|_| AtomicUsize::new(0)).collect::>()); + let events = Arc::new(Mutex::new(Vec::new())); + for index in 0..3 { + let live = live.clone(); + let events = events.clone(); + children.push( + tree.add(ChildSpec::new( + format!("child-{index}"), + &group, + move |ctx| { + let live = live.clone(); + let events = events.clone(); + async move { + assert_eq!(live[index].fetch_add(1, Ordering::SeqCst), 0); + Ok::<_, String>(Probe { + index, + generation: ctx.generation() as usize, + value: 0, + live, + events, + }) + } + }, + )) + .unwrap(), + ); + } + let supervisor = tree.start().await.unwrap(); + // Two rounds, each rebuilding all three children (the first child + // is the origin, so RestForOne rebuilds the whole tail too). + for round in 1..=2u64 { + assert!(children[0].current().unwrap().request(Crash).await.is_err()); + for child in &children { + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(round)) + .await + .unwrap() + .unwrap_or_else(|e| { + panic!( + "{strategy:?} round {round}: {e:?} {:?}", + supervisor.failure() + ) + }); + } + assert!( + supervisor.failure().is_none(), + "{strategy:?}: round {round} charged per rebuilt child: {:?}", + supervisor.failure() + ); + } + assert!(children + .iter() + .all(|child| child.snapshot().generation == 3)); + // The third failure is the third restart action in the window. + assert!(children[0].current().unwrap().request(Crash).await.is_err()); + spawned_rt::tasks::timeout(LIMIT, supervisor.join()) + .await + .unwrap(); + let failure = supervisor.failure().unwrap(); + assert!( + failure.contains("restart budget exhausted by child-0"), + "{strategy:?}: {failure}" ); + assert!(children + .iter() + .all(|child| child.snapshot().generation == 3)); + assert!(system.shutdown(LIMIT).await.complete); + assert!(live.iter().all(|n| n.load(Ordering::SeqCst) == 0)); } - let supervisor = within!(tree.start()).await.unwrap(); - assert!(children[0].current().unwrap().request(Crash).await.is_err()); - spawned_rt::tasks::timeout(LIMIT, supervisor.join()) - .await - .unwrap(); - assert!(supervisor.failure().unwrap().contains("budget exhausted")); - assert!(system.shutdown(LIMIT).await.complete); - assert!(live.iter().all(|n| n.load(Ordering::SeqCst) == 0)); } // Not simulated: children meet at a std::sync::Barrier, which needs OS threads. @@ -583,7 +648,11 @@ async fn upstream_failure_supersedes_a_round_while_a_dependent_factory_is_pendin } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn ready_dependency_failure_rolls_back_slow_initial_tree_startup() { +/// S1: a ready dependency's exit during the initial start is deferred: the +/// start continues (v0.7 rolled back at once), the dependency is not replaced +/// alone, and when a later first start fails the deferred exit is part of the +/// rollback. +async fn ready_dependency_exit_is_deferred_during_slow_initial_tree_startup() { for strategy in [ SupervisorStrategy::OneForAll, SupervisorStrategy::RestForOne, @@ -620,21 +689,41 @@ async fn ready_dependency_failure_rolls_back_slow_initial_tree_startup() { }) } }) - .startup_timeout(Duration::from_secs(30)), + .startup_timeout(if index == 2 { + Duration::from_millis(500) + } else { + Duration::from_secs(30) + }), ) .unwrap(), ); } - let startup = spawned_rt::tasks::spawn(tree.start()); + let finished = Arc::new(AtomicBool::new(false)); + let flag = finished.clone(); + let startup = spawned_rt::tasks::spawn(async move { + let result = tree.start().await; + flag.store(true, Ordering::SeqCst); + result + }); spawned_rt::tasks::timeout(LIMIT, entered.notified()) .await .unwrap(); assert!(children[0].current().unwrap().request(Crash).await.is_err()); - assert!(spawned_rt::tasks::timeout(LIMIT, startup) + spawned_rt::tasks::sleep(Duration::from_millis(100)).await; + assert!( + !finished.load(Ordering::SeqCst), + "{strategy:?}: start rolled back at once" + ); + match spawned_rt::tasks::timeout(LIMIT, startup) .await .unwrap() .unwrap() - .is_err()); + { + Err(SupervisorError::FailedToStartChild { child, .. }) => { + assert_eq!(child, "startup-dependency-2") + } + other => panic!("{strategy:?}: {:?}", other.err()), + } assert_eq!( children[0].snapshot().generation, 1, @@ -708,7 +797,14 @@ impl Actor for MaybeStop { } } impl Handler for MaybeStop { - async fn handle(&mut self, _: CrashPlain, _: &Context) { - panic!("injected cohort failure"); + async fn handle( + &mut self, + _: CrashPlain, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("injected cohort failure"); + } } } diff --git a/concurrency/tests/supervision_trace.rs b/concurrency/tests/supervision_trace.rs index 9d33e28b6..2fb664a8f 100644 --- a/concurrency/tests/supervision_trace.rs +++ b/concurrency/tests/supervision_trace.rs @@ -26,17 +26,15 @@ use std::{ const LIMIT: Duration = Duration::from_secs(4); fn config(max_restarts: usize) -> SupervisorConfig { - SupervisorConfig { - max_restarts, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(8), - jitter: false, - seed: Some(7), - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .max_restarts(max_restarts) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(8)) + .jitter(false) + .seed(7), + ) } fn mode_name(mode: RestartMode) -> &'static str { @@ -101,9 +99,15 @@ impl Message for HoldForDrain { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: HoldForDrain, _: &Context) { + async fn handle( + &mut self, + msg: HoldForDrain, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { msg.0.send(()).unwrap(); let _ = msg.1.await; + reply.reply(()); } } struct DropBomb; @@ -116,7 +120,12 @@ impl Drop for DropBomb { } } impl Handler for Worker { - async fn handle(&mut self, _: DropBomb, _: &Context) { + async fn handle( + &mut self, + _: DropBomb, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("discarded message must not be handled"); } } @@ -146,11 +155,15 @@ async fn trace_queued_destructor_panic_restarts_transient_child() { .unwrap() .unwrap(); assert_ne!(replacement.id(), actor.id()); + // The final reason stays Normal; the destructor panic after the death + // point is a cleanup failure, which restarts a Transient child. assert_eq!( child.snapshot().last_exit, - Some(spawned_concurrency::ExitReason::Panic( - "queued message destructor: injected queued destructor panic".into() - )) + Some(spawned_concurrency::ExitReason::Normal) + ); + assert_eq!( + child.snapshot().last_error.as_deref(), + Some("injected queued destructor panic") ); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( @@ -163,7 +176,7 @@ async fn trace_queued_destructor_panic_restarts_transient_child() { } #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn trace_queued_destructor_failure_preserves_shutdown() { +async fn trace_queued_destructor_failure_after_shutdown_restarts_transient_child() { use spawned_concurrency::ExitReason; let system = ActorSystem::new().unwrap(); let group = system.async_group("trace-drain-shutdown", 1).unwrap(); @@ -181,18 +194,25 @@ async fn trace_queued_destructor_failure_preserves_shutdown() { .unwrap() .unwrap(); actor.send(DropBomb).unwrap(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); release.send(()).unwrap(); - wait_state(&child, ChildState::Stopped).await; - assert_eq!(child.snapshot().generation, 1); + // v0.8: the reason stays Shutdown, but the cleanup failure restarts a + // Transient child (the supervisor is not closing). + let replacement = spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert_ne!(replacement.id(), actor.id()); assert_eq!(child.snapshot().last_exit, Some(ExitReason::Shutdown)); assert_eq!( - actor.child_handle().mailbox_cleanup_failure().as_deref(), + actor.child_handle().cleanup_failure().as_deref(), Some("injected queued destructor panic") ); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( - "queued_drop_preserves_shutdown", + "queued_drop_shutdown_restarts", supervisor.trace_id(), &[RestartMode::Transient], 5, @@ -200,18 +220,35 @@ async fn trace_queued_destructor_failure_preserves_shutdown() { assert!(system.shutdown(LIMIT).await.complete); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected child failure"); } } impl Handler for Worker { - async fn handle(&mut self, _: Quit, ctx: &Context) { + async fn handle( + &mut self, + _: Quit, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop(); + reply.reply(()); } } impl Handler for Worker { - async fn handle(&mut self, msg: ExitWith, ctx: &Context) { + async fn handle( + &mut self, + msg: ExitWith, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.stop_with_reason(msg.0); + reply.reply(()); } } @@ -354,18 +391,106 @@ async fn trace_external_shutdown_of_permanent_child_restarts() { let a = tree.add(worker("a", &group).restart(modes[0])).unwrap(); let b = tree.add(worker("b", &group).restart(modes[1])).unwrap(); let supervisor = within!(tree.start()).await.unwrap(); - a.current().unwrap().child_handle().shutdown(); + a.current() + .unwrap() + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); spawned_rt::tasks::timeout(LIMIT, a.wait_for_generation(1)) .await .unwrap() .unwrap(); - b.current().unwrap().child_handle().shutdown(); + b.current() + .unwrap() + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); wait_state(&b, ChildState::Stopped).await; assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("external_shutdown", supervisor.trace_id(), &modes, 5); assert!(system.shutdown(LIMIT).await.complete); } +/// S1: a Permanent child whose first construction fails aborts the start (v0.7 +/// retried it); the model's Decide fails because the child never completed +/// its first start. +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_first_start_factory_error_fails_start() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-factory-first", 1).unwrap(); + let modes = [RestartMode::Permanent, RestartMode::Permanent]; + let mut tree = system.supervisor(config(5)).unwrap(); + tree.add(worker("a", &group).restart(modes[0])).unwrap(); + tree.add( + ChildSpec::new("flaky", &group, |_| async { + Err::("first construction fails") + }) + .restart(modes[1]), + ) + .unwrap(); + let trace_id = tree.trace_id(); + assert!(spawned_rt::tasks::timeout(LIMIT, tree.start()) + .await + .unwrap() + .is_err()); + assert!(system.shutdown(LIMIT).await.complete); + write_trace("first_start_factory_error", trace_id, &modes, 5); +} + +/// S1: a ready child's exit during the initial start is deferred (DeferExit) +/// and processed once the tree commits ready (Undefer, then the restart). +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_ready_exit_deferred_during_startup() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-deferred", 2).unwrap(); + let modes = [RestartMode::Permanent, RestartMode::Permanent]; + let mut tree = system.supervisor(config(5)).unwrap(); + let a = tree.add(worker("a", &group).restart(modes[0])).unwrap(); + let entered = Arc::new(tokio::sync::Notify::new()); + let release = Arc::new(tokio::sync::Notify::new()); + let (e, r) = (entered.clone(), release.clone()); + tree.add( + ChildSpec::new("b", &group, move |ctx| { + let (e, r) = (e.clone(), r.clone()); + async move { + if ctx.generation() == 1 { + e.notify_one(); + r.notified().await; + } + Ok::<_, String>(Worker) + } + }) + .restart(modes[1]), + ) + .unwrap(); + let trace_id = tree.trace_id(); + let start = spawned_rt::tasks::spawn(tree.start()); + spawned_rt::tasks::timeout(LIMIT, entered.notified()) + .await + .unwrap(); + crash(&a).await; + // The exit is recorded before the start completes. + spawned_rt::tasks::timeout(LIMIT, async { + while !lifecycle_trace::recorded(trace_id, "DeferExit") { + spawned_rt::tasks::sleep(Duration::from_millis(1)).await; + } + }) + .await + .unwrap(); + release.notify_one(); + let supervisor = spawned_rt::tasks::timeout(LIMIT, start) + .await + .unwrap() + .unwrap() + .unwrap(); + spawned_rt::tasks::timeout(LIMIT, a.wait_for_generation(1)) + .await + .unwrap() + .unwrap(); + assert!(supervisor.shutdown(LIMIT).await.complete); + write_trace("ready_exit_deferred", trace_id, &modes, 5); + assert!(system.shutdown(LIMIT).await.complete); +} + +/// A replacement whose construction fails is retried within the budget. #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] async fn trace_factory_error_then_success() { let system = ActorSystem::new().unwrap(); @@ -374,19 +499,25 @@ async fn trace_factory_error_then_success() { let calls = Arc::new(AtomicUsize::new(0)); let mut tree = system.supervisor(config(5)).unwrap(); let count = calls.clone(); - tree.add(ChildSpec::new("flaky", &group, move |_| { - let attempt = count.fetch_add(1, Ordering::SeqCst); - async move { - if attempt == 0 { - Err("first construction fails".to_string()) - } else { - Ok(Worker) + let flaky = tree + .add(ChildSpec::new("flaky", &group, move |_| { + let attempt = count.fetch_add(1, Ordering::SeqCst); + async move { + if attempt == 1 { + Err("replacement construction fails".to_string()) + } else { + Ok(Worker) + } } - } - })) - .unwrap(); + })) + .unwrap(); let supervisor = within!(tree.start()).await.unwrap(); - assert_eq!(calls.load(Ordering::SeqCst), 2); + crash(&flaky).await; + spawned_rt::tasks::timeout(LIMIT, flaky.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 3); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace( "factory_error_then_success", @@ -501,7 +632,12 @@ impl Actor for ExitOnceDuringRestart { } } impl Handler for ExitOnceDuringRestart { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("injected child failure"); } } @@ -561,26 +697,58 @@ impl Actor for SlowInit { } } } +/// S1: a first start that misses startup_timeout aborts the start, whatever +/// the restart type (v0.7 retried a Transient child). #[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] -async fn trace_startup_timeout_retries() { +async fn trace_first_start_timeout_fails_start() { let system = ActorSystem::new().unwrap(); - let group = system.async_group("trace-timeout", 1).unwrap(); + let group = system.async_group("trace-timeout-first", 1).unwrap(); let modes = [RestartMode::Transient]; let mut tree = system.supervisor(config(3)).unwrap(); tree.add( - ChildSpec::new("slow-init", &group, |ctx| async move { - Ok::<_, String>(SlowInit { - first: ctx.generation() == 1, - }) + ChildSpec::new("slow-init", &group, |_| async move { + Ok::<_, String>(SlowInit { first: true }) }) .restart(modes[0]) .startup_timeout(Duration::from_millis(20)), ) .unwrap(); + let trace_id = tree.trace_id(); + assert!(spawned_rt::tasks::timeout(LIMIT, tree.start()) + .await + .unwrap() + .is_err()); + assert!(system.shutdown(LIMIT).await.complete); + write_trace("first_start_timeout", trace_id, &modes, 3); +} + +/// A replacement that misses startup_timeout is retried. +#[spawned_macros::sim_test(flavor = "multi_thread", worker_threads = 2)] +async fn trace_startup_timeout_retries() { + let system = ActorSystem::new().unwrap(); + let group = system.async_group("trace-timeout", 1).unwrap(); + let modes = [RestartMode::Transient]; + let mut tree = system.supervisor(config(3)).unwrap(); + let child = tree + .add( + ChildSpec::new("slow-init", &group, |ctx| async move { + Ok::<_, String>(SlowInit { + first: ctx.generation() == 2, + }) + }) + .restart(modes[0]) + .startup_timeout(Duration::from_millis(20)), + ) + .unwrap(); let supervisor = spawned_rt::tasks::timeout(LIMIT, tree.start()) .await .unwrap() .unwrap(); + child.current().unwrap().child_handle().kill(); + spawned_rt::tasks::timeout(LIMIT, child.wait_for_generation(2)) + .await + .unwrap() + .unwrap(); assert!(supervisor.shutdown(LIMIT).await.complete); write_trace("startup_timeout", supervisor.trace_id(), &modes, 3); assert!(system.shutdown(LIMIT).await.complete); @@ -590,6 +758,11 @@ struct SlowStop { release: Option>, } impl Actor for SlowStop { + async fn started(&mut self, ctx: &Context) { + // v0.8: stopped() runs on a parent's shutdown only for a trapping + // actor (E3, decision 2). + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { if let Some(release) = self.release.take() { let _ = release.await; @@ -682,7 +855,9 @@ async fn trace_randomized_lifecycles() { 1 => { let _ = actor.send(Quit); } - _ => actor.child_handle().shutdown(), + _ => actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown), } } spawned_rt::tasks::sleep(Duration::from_millis(rng.next(4))).await; diff --git a/concurrency/tests/supervisor_policies.rs b/concurrency/tests/supervisor_policies.rs index bda79d4b6..8960a0740 100644 --- a/concurrency/tests/supervisor_policies.rs +++ b/concurrency/tests/supervisor_policies.rs @@ -28,13 +28,25 @@ impl Message for Crash { struct Probe(u64); impl Actor for Probe {} impl Handler for Probe { - async fn handle(&mut self, _: Generation, _: &Context) -> u64 { - self.0 + async fn handle( + &mut self, + _: Generation, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0); } } impl Handler for Probe { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("generated supervisor failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("generated supervisor failure"); + } } } #[derive(Clone, Debug)] @@ -91,6 +103,7 @@ fn apply( SupervisorStrategy::OneForOne => index == origin, SupervisorStrategy::OneForAll => true, SupervisorStrategy::RestForOne => index >= origin, + other => unreachable!("strategy {other:?} not modelled"), }; if affected && child.live { if modes[index] == RestartMode::Temporary { @@ -101,7 +114,27 @@ fn apply( } } } -fn unsettled(actual: &[ChildSnapshot], expected: &[Expected]) -> Vec { +/// The snapshot fields the checker reads (`ChildSnapshot` is +/// `#[non_exhaustive]`, so the negative test below builds these instead). +#[derive(Debug)] +struct Seen { + name: String, + state: ChildState, + generation: u64, +} +impl From for Seen { + fn from(s: ChildSnapshot) -> Self { + Seen { + name: s.name, + state: s.state, + generation: s.generation, + } + } +} +fn seen(children: &[SupervisedChild]) -> Vec { + children.iter().map(|c| c.snapshot().into()).collect() +} +fn unsettled(actual: &[Seen], expected: &[Expected]) -> Vec { assert_eq!(actual.len(), expected.len()); let mut pending = Vec::new(); // Do not short-circuit: a later child may have been over-reconstructed even @@ -131,7 +164,7 @@ fn unsettled(actual: &[ChildSnapshot], expected: &[Expected]) -> Vec { async fn observe(children: &[SupervisedChild], expected: &[Expected]) { let outcome = spawned_rt::tasks::timeout(LIMIT, async { loop { - let actual: Vec<_> = children.iter().map(SupervisedChild::snapshot).collect(); + let actual = seen(children); let pending = unsettled(&actual, expected); if pending.is_empty() { break; @@ -168,7 +201,7 @@ async fn observe(children: &[SupervisedChild], expected: &[Expected]) { }) .await; if outcome.is_err() { - let actual: Vec<_> = children.iter().map(SupervisedChild::snapshot).collect(); + let actual = seen(children); // Preserve the concrete over-reconstruction diagnosis at the watchdog // boundary too, rather than hiding it behind a generic timeout. let pending = unsettled(&actual, expected); @@ -201,19 +234,19 @@ async fn scenario( let system = ActorSystem::new().unwrap(); let group = system.async_group("policy-oracle", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 128, - restart_window: Duration::from_secs(3600), - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - seed: Some(42), - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(128) + .restart_window(Duration::from_secs(3600)) + .backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false) + .seed(42), + ), + ) .unwrap(); let mut children = Vec::new(); for (index, restart) in modes.iter().enumerate() { @@ -258,7 +291,7 @@ async fn scenario( Action::Panic => { assert!(matches!( actor.request_with_timeout(Crash, LIMIT).await, - Err(ActorError::ActorStopped) + Err(ActorError::ActorStopped(ExitReason::Panic(_))) )); let reason = within!(actor.wait_exit()).await; assert!(matches!(reason, ExitReason::Panic(_))); @@ -270,7 +303,9 @@ async fn scenario( false } Action::Shutdown => { - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(within!(actor.wait_exit()).await, ExitReason::Shutdown); false } @@ -337,13 +372,10 @@ fn retired_temporary_and_transient_slots_stay_retired_across_later_recoveries() #[test] #[should_panic(expected = "unexpected extra reconstruction: child-1")] fn a_waiting_earlier_child_cannot_hide_a_later_extra_reconstruction() { - let snapshot = |name: &str, state, generation| ChildSnapshot { + let snapshot = |name: &str, state, generation| Seen { name: name.into(), state, generation, - actor_id: None, - last_exit: None, - last_error: None, }; let actual = [ snapshot("child-0", ChildState::Backoff, 1), diff --git a/concurrency/tests/support/failure_actor.rs b/concurrency/tests/support/failure_actor.rs index a7db6149a..6a6469458 100644 --- a/concurrency/tests/support/failure_actor.rs +++ b/concurrency/tests/support/failure_actor.rs @@ -159,40 +159,50 @@ impl spawned_concurrency::tasks::Handler for AsyncWorker { async fn handle( &mut self, command: Command, + reply: spawned_concurrency::PendingReply<'_, Reply>, _: &spawned_concurrency::tasks::Context, - ) -> Reply { - match command { - Command::Payload { id, payload } => { - assert_eq!(payload.bytes[0], (id % 251) as u8); - self.0.execute(id) - } - Command::Read => Reply::Values(self.0.values.clone()), - Command::Panic => panic!("controlled handler panic"), - Command::Append { - id, - before, - after, - panic_after, - } => { - if let Some(gate) = before { - gate.wait().await; + ) { + let __reply_value: Reply = { + match command { + Command::Payload { id, payload } => { + assert_eq!(payload.bytes[0], (id % 251) as u8); + self.0.execute(id) } - let result = self.0.execute(id); - if let Some(gate) = after { - gate.wait().await; + Command::Read => Reply::Values(self.0.values.clone()), + Command::Panic => panic!("controlled handler panic"), + Command::Append { + id, + before, + after, + panic_after, + } => { + if let Some(gate) = before { + gate.wait().await; + } + let result = self.0.execute(id); + if let Some(gate) = after { + gate.wait().await; + } + assert!(!panic_after, "controlled panic after effect"); + result } - assert!(!panic_after, "controlled panic after effect"); - result } - } + }; + reply.reply(__reply_value); } } #[cfg(feature = "async")] impl spawned_concurrency::tasks::Handler for AsyncWorker { - async fn handle(&mut self, down: Down, _: &spawned_concurrency::tasks::Context) { + async fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &spawned_concurrency::tasks::Context, + ) { if let Some(sender) = &self.0.down { let _ = sender.send(down); } + reply.reply(()); } } pub struct NativeWorker(pub Worker); @@ -213,39 +223,49 @@ impl spawned_concurrency::threads::Handler for NativeWorker { fn handle( &mut self, command: Command, + reply: spawned_concurrency::PendingReply<'_, Reply>, _: &spawned_concurrency::threads::Context, - ) -> Reply { - match command { - Command::Payload { id, payload } => { - assert_eq!(payload.bytes[0], (id % 251) as u8); - self.0.execute(id) - } - Command::Read => Reply::Values(self.0.values.clone()), - Command::Panic => panic!("controlled handler panic"), - Command::Append { - id, - before, - after, - panic_after, - } => { - if let Some(gate) = before { - futures::executor::block_on(gate.wait()); + ) { + let __reply_value: Reply = { + match command { + Command::Payload { id, payload } => { + assert_eq!(payload.bytes[0], (id % 251) as u8); + self.0.execute(id) } - let result = self.0.execute(id); - if let Some(gate) = after { - futures::executor::block_on(gate.wait()); + Command::Read => Reply::Values(self.0.values.clone()), + Command::Panic => panic!("controlled handler panic"), + Command::Append { + id, + before, + after, + panic_after, + } => { + if let Some(gate) = before { + futures::executor::block_on(gate.wait()); + } + let result = self.0.execute(id); + if let Some(gate) = after { + futures::executor::block_on(gate.wait()); + } + assert!(!panic_after, "controlled panic after effect"); + result } - assert!(!panic_after, "controlled panic after effect"); - result } - } + }; + reply.reply(__reply_value); } } impl spawned_concurrency::threads::Handler for NativeWorker { - fn handle(&mut self, down: Down, _: &spawned_concurrency::threads::Context) { + fn handle( + &mut self, + down: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &spawned_concurrency::threads::Context, + ) { if let Some(sender) = &self.0.down { let _ = sender.send(down); } + reply.reply(()); } } #[derive(Clone)] @@ -254,10 +274,12 @@ pub enum Handle { Async(spawned_concurrency::tasks::ActorRef), Native(spawned_concurrency::threads::ActorRef), } +/// A raw request sent without a deadline; [`Response::wait`] is the caller +/// giving up after `duration` (dropping the receiver abandons the request). pub enum Response { #[cfg(feature = "async")] - Async(tokio::sync::oneshot::Receiver), - Native(mpsc::Receiver), + Async(spawned_concurrency::ReplyReceiver), + Native(spawned_concurrency::ReplyReceiver), } impl Response { pub fn wait( @@ -267,13 +289,23 @@ impl Response { ) -> Result { match self { #[cfg(feature = "async")] - Self::Async(receiver) => _rt.block_on( - spawned_concurrency::Response::from_with_timeout(Ok(receiver), duration), - ), - Self::Native(receiver) => receiver.recv_timeout(duration).map_err(|e| match e { - mpsc::RecvTimeoutError::Timeout => ActorError::RequestTimeout, - mpsc::RecvTimeoutError::Disconnected => ActorError::ActorStopped, + Self::Async(receiver) => _rt.block_on(async { + tokio::time::timeout(duration, receiver) + .await + .unwrap_or(Err(ActorError::RequestTimeout)) }), + Self::Native(mut receiver) => { + let give_up = std::time::Instant::now() + duration; + loop { + if let Some(result) = receiver.try_recv() { + return result; + } + if std::time::Instant::now() >= give_up { + return Err(ActorError::RequestTimeout); + } + std::thread::sleep(Duration::from_millis(1)); + } + } } } } @@ -303,8 +335,12 @@ impl Handle { pub fn raw(&self, command: Command) -> Result { match self { #[cfg(feature = "async")] - Self::Async(actor) => actor.request_raw(command).map(Response::Async), - Self::Native(actor) => actor.request_raw(command).map(Response::Native), + Self::Async(actor) => actor + .send_request(command, spawned_concurrency::Deadline::Never) + .map(Response::Async), + Self::Native(actor) => actor + .send_request(command, spawned_concurrency::Deadline::Never) + .map(Response::Native), } } pub fn request( diff --git a/concurrency/tests/support/store_actor.rs b/concurrency/tests/support/store_actor.rs index 4d5b3091f..7c8a27b19 100644 --- a/concurrency/tests/support/store_actor.rs +++ b/concurrency/tests/support/store_actor.rs @@ -115,13 +115,14 @@ impl spawned_concurrency::tasks::Handler for AsyncStore { async fn handle( &mut self, op: StoreOp, + reply: spawned_concurrency::PendingReply<'_, Option>, _: &spawned_concurrency::tasks::Context, - ) -> Option { + ) { // Yield inside the turn so concurrent requests really queue up. if op.key().1.is_multiple_of(2) { tokio::task::yield_now().await; } - self.0.apply(op) + reply.reply(self.0.apply(op)); } } @@ -133,9 +134,10 @@ impl spawned_concurrency::threads::Handler for NativeStore { fn handle( &mut self, op: StoreOp, + reply: spawned_concurrency::PendingReply<'_, Option>, _: &spawned_concurrency::threads::Context, - ) -> Option { - self.0.apply(op) + ) { + reply.reply(self.0.apply(op)); } } diff --git a/concurrency/tests/tree_trace.rs b/concurrency/tests/tree_trace.rs index 28d6feaa9..85fa6ffb3 100644 --- a/concurrency/tests/tree_trace.rs +++ b/concurrency/tests/tree_trace.rs @@ -23,22 +23,25 @@ impl Message for Crash { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Crash, _: &Context) { + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { panic!("nested trace failure"); } } fn config(strategy: SupervisorStrategy) -> SupervisorConfig { - SupervisorConfig { - strategy, - max_restarts: 10, - backoff: Backoff { - initial: Duration::from_millis(1), - maximum: Duration::from_millis(2), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - } + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(10) + .backoff( + Backoff::default() + .initial(Duration::from_millis(1)) + .maximum(Duration::from_millis(2)) + .jitter(false), + ) } #[path = "support/tree_record.rs"] mod tree_record; @@ -59,10 +62,7 @@ async fn mixed_strategy_three_level_shutdown_and_subtree_replacement() { .add_supervisor(SupervisorSpec::new( "branch", &group, - SupervisorConfig { - max_restarts: 0, - ..config(SupervisorStrategy::RestForOne) - }, + config(SupervisorStrategy::RestForOne).max_restarts(0), move |middle| { let placement = placement.clone(); let expose = expose.clone(); @@ -70,10 +70,7 @@ async fn mixed_strategy_three_level_shutdown_and_subtree_replacement() { middle.add_supervisor(SupervisorSpec::new( "nested", &control, - SupervisorConfig { - max_restarts: 0, - ..config(SupervisorStrategy::OneForAll) - }, + config(SupervisorStrategy::OneForAll).max_restarts(0), move |inner| { *expose.lock().unwrap() = Some(inner.add(ChildSpec::new("leaf", &placement, |_| async { diff --git a/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs b/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs index 26f3fad12..3870b3e95 100644 --- a/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs +++ b/concurrency/tests/ui-pass/actor_tasks_lifecycle.rs @@ -25,11 +25,11 @@ mod asynchronous { async fn halt(&mut self, _ctx: &Context) {} #[exit_received] async fn trapped(&mut self, _exit: Exit, _ctx: &Context) {} - #[handler] + #[request_handler] async fn ping(&mut self, msg: Ping, _ctx: &Context) -> u32 { msg.0 + 1 } - #[handler] + #[request_handler] fn pong( &mut self, msg: Pong, diff --git a/concurrency/tests/ui-pass/api_08.rs b/concurrency/tests/ui-pass/api_08.rs new file mode 100644 index 000000000..f7faeff63 --- /dev/null +++ b/concurrency/tests/ui-pass/api_08.rs @@ -0,0 +1,60 @@ +// The 0.8 request, deadline, notice and configuration API compiles as +// documented in MIGRATION's 0.7 to 0.8 call-site table. +use spawned_concurrency::message::Message; +use spawned_concurrency::supervision::{Backoff, SupervisorConfig, SupervisorStrategy}; +use spawned_concurrency::tasks::{Actor, ActorRef, Context, Handler, PendingReply, Recipient}; +use spawned_concurrency::{ + ActorError, ActorId, Deadline, Down, Exit, ExitReason, ReplyTo, Response, + DEFAULT_REQUEST_TIMEOUT, +}; +use std::time::Duration; + +struct Get; +impl Message for Get { + type Result = u32; +} +struct Counter; +impl Actor for Counter {} +impl Handler for Counter { + async fn handle(&mut self, _: Get, reply: PendingReply<'_, u32>, _: &Context) { + // A deadline forwards as is: `other.send_request(msg, reply.deadline())`. + let _: Deadline = reply.deadline(); + reply.reply(1); + } +} + +fn requests(actor: &ActorRef, recipient: &Recipient) -> Result<(), ActorError> { + let _rx = actor.send_request(Get, Duration::from_secs(1))?; + let _rx = actor.send_request(Get, Deadline::Never)?; + let _rx = actor.send_request(Get, spawned_concurrency::Instant::now())?; + let rx = recipient.send_request(Get, Deadline::after(DEFAULT_REQUEST_TIMEOUT))?; + let _: Deadline = rx.deadline(); + let _response: Response = Response::from(actor.send_request(Get, DEFAULT_REQUEST_TIMEOUT)); + let _ctx: Context = actor.context(); + actor.child_handle().stop_with_reason(ExitReason::Shutdown); + Ok(()) +} + +fn token(token: &ReplyTo) { + token.shorten_deadline(Duration::from_millis(5)); + let _: Deadline = token.deadline(); +} + +fn notices(down: Down, exit: Exit) { + let Down { actor, reason, .. } = down; + let _: (ActorId, ExitReason) = (actor, reason); + let Exit { from, .. } = exit; + let _: ActorId = from; + let _ = Exit::new(ActorId::EXTERNAL, ExitReason::Normal); +} + +fn config() -> SupervisorConfig { + SupervisorConfig::new() + .strategy(SupervisorStrategy::RestForOne) + .max_restarts(3) + .backoff(Backoff::new().jitter(false).seed(7)) +} + +fn main() { + let _ = (requests, token, notices, config); +} diff --git a/concurrency/tests/ui-pass/deferred_handlers.rs b/concurrency/tests/ui-pass/deferred_handlers.rs new file mode 100644 index 000000000..719f51295 --- /dev/null +++ b/concurrency/tests/ui-pass/deferred_handlers.rs @@ -0,0 +1,71 @@ +// `#[deferred_handler]` in both modes: answer in place, or defer and answer +// from a later turn; a `ReplyTo` is storable and `Send + 'static`. +use spawned_concurrency::{actor, protocol, DeferError, PendingReply, ReplyTo, Response}; + +#[protocol] +pub trait WaitProtocol: Send + Sync { + fn wait(&self) -> Response; + fn release(&self, value: u64); +} +use wait_protocol::{Release, Wait}; + +mod tasks_mode { + use super::*; + use spawned_concurrency::tasks::{Actor, Context, Handler}; + pub struct Gate { + waiters: Vec>, + ready: Option, + } + #[actor(protocol = WaitProtocol)] + impl Gate { + #[deferred_handler] + async fn wait(&mut self, _msg: Wait, reply: PendingReply<'_, u64>, _ctx: &Context) { + if let Some(value) = self.ready { + reply.reply(value); + return; + } + match reply.defer() { + Ok(token) => self.waiters.push(token), + Err(DeferError::Overloaded | DeferError::Closed | DeferError::NoCaller) => {} + // `DeferError` is `#[non_exhaustive]`. + Err(_) => {} + } + } + #[send_handler] + async fn release(&mut self, msg: Release, _ctx: &Context) { + self.ready = Some(msg.value); + for token in self.waiters.drain(..) { + token.reply(msg.value); + } + } + } +} + +mod threads_mode { + use super::*; + use spawned_concurrency::threads::{Actor, Context, Handler}; + pub struct Gate { + waiters: Vec>, + } + #[actor(protocol = WaitProtocol)] + impl Gate { + #[deferred_handler] + fn wait(&mut self, _msg: Wait, reply: PendingReply<'_, u64>, _ctx: &Context) { + if let Ok(token) = reply.defer() { + self.waiters.push(token); + } + } + #[send_handler] + fn release(&mut self, msg: Release, _ctx: &Context) { + for token in self.waiters.drain(..) { + token.reply(msg.value); + } + } + } +} + +fn assert_storable() {} + +fn main() { + assert_storable::>(); +} diff --git a/concurrency/tests/ui-pass/job_api.rs b/concurrency/tests/ui-pass/job_api.rs new file mode 100644 index 000000000..03423bf54 --- /dev/null +++ b/concurrency/tests/ui-pass/job_api.rs @@ -0,0 +1,71 @@ +// The 0.8 job API: one call per shape, options first, the work receives +// its `JobCtx`, `JobOutcome` is a `Result`, `PipeError` is the enum. +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{ + Actor, Context, Deadline, Handler, JobCtx, JobError, JobOptions, JobOutcome, PendingReply, + PipeError, PipeReplyError, ReplyTo, +}; +use spawned_concurrency::Instant; +use std::num::NonZeroU16; +use std::time::Duration; + +struct Owner; +impl Actor for Owner {} + +struct Done(JobOutcome); +impl Message for Done { + type Result = (); +} +struct Answered(JobOutcome, ReplyTo); +impl Message for Answered { + type Result = (); +} + +impl Handler for Owner { + async fn handle(&mut self, msg: Done, _: PendingReply<'_, ()>, _: &Context) { + // A plain `Result`: `?`-style handling and wildcard error arms. + let _value: Result = msg.0; + } +} +impl Handler for Owner { + async fn handle(&mut self, msg: Answered, _: PendingReply<'_, ()>, _: &Context) { + let Answered(outcome, token) = msg; + let _ = token.reply(outcome.unwrap_or(0)); + } +} + +fn spawn_all(ctx: &Context, token: ReplyTo) -> Result<(), PipeError> { + let credits = NonZeroU16::new(4).unwrap(); + ctx.pipe(JobOptions::new(), |_| async { 1 }, Done)?; + ctx.pipe( + JobOptions::new().key("k").emit_credits(credits), + |job: JobCtx| async move { u32::from(job.is_cancelled()) }, + Done, + )?; + ctx.pipe_blocking(JobOptions::new().deadline(Duration::from_secs(1)), |_| 2, Done)?; + ctx.pipe_blocking(JobOptions::new().deadline(Instant::now()), |_| 3, Done)?; + ctx.pipe_blocking(JobOptions::new().deadline(Deadline::Never), |_| 4, Done)?; + if let Ok(permit) = ctx.reserve_job(JobOptions::new()) { + permit.pipe(|_| async { 5 }, Done); + } + let permit = ctx.reserve_blocking_job(JobOptions::default())?; + permit.pipe_blocking(|job| u32::from(job.is_cancelled()), Done); + match ctx.pipe_reply( + JobOptions::new().cancel_on_caller_gone(), + token, + |_| async { 6 }, + Answered, + ) { + Ok(_) => {} + Err(PipeReplyError::Failed { token, error }) => { + let _ = matches!(error, PipeError::Overloaded | PipeError::NativeBudget); + let _ = ctx.pipe_reply_blocking(JobOptions::new(), token, |_| 7, Answered); + } + Err(other) => drop(other.into_token()), + } + Ok(()) +} + +fn main() { + let _ = spawn_all; +} diff --git a/concurrency/tests/ui-pass/protocol_edge_cases.rs b/concurrency/tests/ui-pass/protocol_edge_cases.rs index 80493d82f..b3f3a7342 100644 --- a/concurrency/tests/ui-pass/protocol_edge_cases.rs +++ b/concurrency/tests/ui-pass/protocol_edge_cases.rs @@ -56,7 +56,7 @@ impl Store { fn on_callback(&mut self, msg: http_store::Callback, _: &Context) -> usize { (msg.f)("abc") } - #[handler] + #[request_handler] fn on_total(&mut self, _: http_store::Total, _: &Context) -> u64 { u64::from(self.total) } diff --git a/concurrency/tests/ui-pass/send_handler_renamed.rs b/concurrency/tests/ui-pass/send_handler_renamed.rs new file mode 100644 index 000000000..478572cf2 --- /dev/null +++ b/concurrency/tests/ui-pass/send_handler_renamed.rs @@ -0,0 +1,37 @@ +// `#[send_handler]` in a crate that renames `spawned-concurrency`: the +// local module shadows the original name, so a hard-coded +// `spawned_concurrency::` path in the expansion fails to resolve. +extern crate spawned_concurrency as my_spawned; + +#[allow(dead_code)] +mod spawned_concurrency {} + +use my_spawned::message::Message; +use my_spawned::send_handler; +use my_spawned::tasks::{Actor, Context, Handler}; + +struct Poller; +impl Actor for Poller {} + +struct Tick; +impl Message for Tick { + type Result = (); +} +struct Tock; +impl Message for Tock { + type Result = (); +} + +// A qualified `Handler` path names the reply type through its module. +#[send_handler] +impl my_spawned::tasks::Handler for Poller { + async fn handle(&mut self, _msg: Tick, _ctx: &Context) {} +} + +// A bare `Handler` takes the renamed crate explicitly. +#[send_handler(crate = my_spawned)] +impl Handler for Poller { + async fn handle(&mut self, _msg: Tock, _ctx: &Context) {} +} + +fn main() {} diff --git a/concurrency/tests/ui/actor_async_handler_threads.stderr b/concurrency/tests/ui/actor_async_handler_threads.stderr index e85a6f5d1..b4f66c25b 100644 --- a/concurrency/tests/ui/actor_async_handler_threads.stderr +++ b/concurrency/tests/ui/actor_async_handler_threads.stderr @@ -2,7 +2,7 @@ error[E0053]: method `handle` has an incompatible type for trait --> tests/ui/actor_async_handler_threads.rs:14:5 | 14 | async fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0 } - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found future + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `()`, found future | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> impl Future` + = note: expected signature `fn(&mut Counter, counter_protocol::Read, PendingReply<'_, u64>, &spawned_concurrency::threads::Context) -> ()` + found signature `fn(&mut Counter, counter_protocol::Read, PendingReply<'_, u64>, &spawned_concurrency::threads::Context) -> impl Future` diff --git a/concurrency/tests/ui/actor_exit_with_handler.rs b/concurrency/tests/ui/actor_exit_with_handler.rs index b8cc0ebd6..ec8a0d4ea 100644 --- a/concurrency/tests/ui/actor_exit_with_handler.rs +++ b/concurrency/tests/ui/actor_exit_with_handler.rs @@ -12,7 +12,7 @@ use spawned_concurrency::Exit; #[actor] impl Counter { #[exit_received] - #[handler] + #[send_handler] fn on_exit(&mut self, _exit: Exit, _ctx: &Context) {} } fn main() {} diff --git a/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr b/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr index 15671edd5..fdabe96e5 100644 --- a/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr +++ b/concurrency/tests/ui/actor_handler_ctx_wrong_type.stderr @@ -1,10 +1,10 @@ error[E0308]: mismatched types - --> tests/ui/actor_handler_ctx_wrong_type.rs:14:5 + --> tests/ui/actor_handler_ctx_wrong_type.rs:14:12 | 14 | fn bump(&mut self, msg: Bump, _ctx: &u64) { self.0 += msg.by; } - | ^^ ---- arguments to this method are incorrect - | | - | expected `&u64`, found `&Context` + | ----^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `&u64`, found `&Context` + | | + | arguments to this method are incorrect | = note: expected reference `&u64` found reference `&spawned_concurrency::threads::Context` diff --git a/concurrency/tests/ui/actor_handler_generic.rs b/concurrency/tests/ui/actor_handler_generic.rs index 5754546c4..95359b4c5 100644 --- a/concurrency/tests/ui/actor_handler_generic.rs +++ b/concurrency/tests/ui/actor_handler_generic.rs @@ -10,7 +10,7 @@ pub struct Counter(u64); use spawned_concurrency::threads::{Actor, Context, Handler}; #[actor] impl Counter { - #[handler] + #[send_handler] fn any>(&mut self, _msg: M, _ctx: &Context) {} } fn main() {} diff --git a/concurrency/tests/ui/actor_handler_missing_return.stderr b/concurrency/tests/ui/actor_handler_missing_return.stderr index 9a69cdff2..c535dadd4 100644 --- a/concurrency/tests/ui/actor_handler_missing_return.stderr +++ b/concurrency/tests/ui/actor_handler_missing_return.stderr @@ -1,12 +1,19 @@ -error[E0053]: method `handle` has an incompatible type for trait - --> tests/ui/actor_handler_missing_return.rs:14:57 +error[E0308]: mismatched types + --> tests/ui/actor_handler_missing_return.rs:14:12 | +11 | #[actor] + | -------- arguments to this method are incorrect +... 14 | fn read(&mut self, _msg: Read, _ctx: &Context) {} - | ^ expected `u64`, found `()` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected `u64`, found `()` | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> ()` -help: change the output type to match the trait +note: method `read` modifies its receiver in-place + --> tests/ui/actor_handler_missing_return.rs:14:8 | -14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 {} - | ++++++ +14 | fn read(&mut self, _msg: Read, _ctx: &Context) {} + | ^^^^ this call modifies `self` in-place +note: method defined here + --> src/reply/mod.rs + | + | pub fn reply(mut self, value: R) -> Delivery { + | ^^^^^ diff --git a/concurrency/tests/ui/actor_handler_not_message.stderr b/concurrency/tests/ui/actor_handler_not_message.stderr index 234704c7d..519865696 100644 --- a/concurrency/tests/ui/actor_handler_not_message.stderr +++ b/concurrency/tests/ui/actor_handler_not_message.stderr @@ -44,3 +44,24 @@ note: required by a bound in `spawned_concurrency::threads::Handler` | pub trait Handler: Actor { | ^^^^^^^ required by this bound in `Handler` = note: this error originates in the attribute macro `protocol` (in Nightly builds, run with -Z macro-backtrace for more info) + +error[E0277]: the trait bound `u64: Message` is not satisfied + --> tests/ui/actor_handler_not_message.rs:11:1 + | +11 | #[actor] + | ^^^^^^^^ the trait `Message` is not implemented for `u64` + | +help: the following other types implement trait `Message` + --> tests/ui/actor_handler_not_message.rs:3:1 + | + 3 | #[protocol] + | ^^^^^^^^^^^ + | | + | `Bump` + | `counter_protocol::Read` + | + ::: src/monitor.rs + | + | impl Message for Down { + | ^^^^^^^^^^^^^^^^^^^^^ `Down` + = note: this error originates in the attribute macro `actor` which comes from the expansion of the attribute macro `protocol` (in Nightly builds, run with -Z macro-backtrace for more info) diff --git a/concurrency/tests/ui/actor_handler_removed.rs b/concurrency/tests/ui/actor_handler_removed.rs new file mode 100644 index 000000000..5fd32f5da --- /dev/null +++ b/concurrency/tests/ui/actor_handler_removed.rs @@ -0,0 +1,17 @@ +// `#[handler]` was removed in 0.8: it hid whether a method answers. +use spawned_concurrency::actor; +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{Actor, Context, Handler}; +pub struct Ping; +impl Message for Ping { + type Result = u32; +} +pub struct Counter; +#[actor] +impl Counter { + #[handler] + async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + 1 + } +} +fn main() {} diff --git a/concurrency/tests/ui/actor_handler_removed.stderr b/concurrency/tests/ui/actor_handler_removed.stderr new file mode 100644 index 000000000..534dc6a9c --- /dev/null +++ b/concurrency/tests/ui/actor_handler_removed.stderr @@ -0,0 +1,13 @@ +error: #[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value) + --> tests/ui/actor_handler_removed.rs:12:5 + | +12 | #[handler] + | ^^^^^^^^^^ + +warning: unused imports: `Actor`, `Context`, and `Handler` + --> tests/ui/actor_handler_removed.rs:4:34 + | +4 | use spawned_concurrency::tasks::{Actor, Context, Handler}; + | ^^^^^ ^^^^^^^ ^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/concurrency/tests/ui/actor_handler_wrong_return.stderr b/concurrency/tests/ui/actor_handler_wrong_return.stderr index 3a7b60ff5..51c1a7acc 100644 --- a/concurrency/tests/ui/actor_handler_wrong_return.stderr +++ b/concurrency/tests/ui/actor_handler_wrong_return.stderr @@ -1,13 +1,14 @@ -error[E0053]: method `handle` has an incompatible type for trait +error[E0308]: mismatched types --> tests/ui/actor_handler_wrong_return.rs:14:61 | +11 | #[actor] + | -------- arguments to this method are incorrect +... 14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> String { self.0.to_string() } | ^^^^^^ expected `u64`, found `String` | - = note: expected signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> u64` - found signature `fn(&mut Counter, counter_protocol::Read, &spawned_concurrency::threads::Context) -> String` -help: change the output type to match the trait - | -14 - fn read(&mut self, _msg: Read, _ctx: &Context) -> String { self.0.to_string() } -14 + fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0.to_string() } +note: method defined here + --> src/reply/mod.rs | + | pub fn reply(mut self, value: R) -> Delivery { + | ^^^^^ diff --git a/concurrency/tests/ui/actor_send_handler_returns_value.rs b/concurrency/tests/ui/actor_send_handler_returns_value.rs new file mode 100644 index 000000000..ebdef4ee1 --- /dev/null +++ b/concurrency/tests/ui/actor_send_handler_returns_value.rs @@ -0,0 +1,17 @@ +// A `#[send_handler]` returning a value would silently answer a request. +use spawned_concurrency::actor; +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{Actor, Context, Handler}; +pub struct Ping; +impl Message for Ping { + type Result = u32; +} +pub struct Counter; +#[actor] +impl Counter { + #[send_handler] + async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + 1 + } +} +fn main() {} diff --git a/concurrency/tests/ui/actor_send_handler_returns_value.stderr b/concurrency/tests/ui/actor_send_handler_returns_value.stderr new file mode 100644 index 000000000..ccc8e2590 --- /dev/null +++ b/concurrency/tests/ui/actor_send_handler_returns_value.stderr @@ -0,0 +1,13 @@ +error: #[send_handler] methods return `()`; use #[request_handler] to answer a request with a value + --> tests/ui/actor_send_handler_returns_value.rs:13:67 + | +13 | async fn ping(&mut self, _msg: Ping, _ctx: &Context) -> u32 { + | ^^^ + +warning: unused imports: `Actor`, `Context`, and `Handler` + --> tests/ui/actor_send_handler_returns_value.rs:4:34 + | +4 | use spawned_concurrency::tasks::{Actor, Context, Handler}; + | ^^^^^ ^^^^^^^ ^^^^^^^ + | + = note: `#[warn(unused_imports)]` (part of `#[warn(unused)]`) on by default diff --git a/concurrency/tests/ui/actor_sync_handler_tasks.stderr b/concurrency/tests/ui/actor_sync_handler_tasks.stderr index f77b3e28e..08b2ced99 100644 --- a/concurrency/tests/ui/actor_sync_handler_tasks.stderr +++ b/concurrency/tests/ui/actor_sync_handler_tasks.stderr @@ -1,12 +1,12 @@ -error[E0277]: `u64` is not a future - --> tests/ui/actor_sync_handler_tasks.rs:14:61 +error[E0277]: `()` is not a future + --> tests/ui/actor_sync_handler_tasks.rs:14:57 | 14 | fn read(&mut self, _msg: Read, _ctx: &Context) -> u64 { self.0 } - | ^^^ `u64` is not a future + | ^ `()` is not a future | - = help: the trait `Future` is not implemented for `u64` + = help: the trait `Future` is not implemented for `()` note: required by a bound in `spawned_concurrency::tasks::Handler::handle::{anon_assoc#0}` --> src/tasks/actor.rs | - | fn handle(&mut self, msg: M, ctx: &Context) -> impl Future + Send; - | ^^^^^^^^^^^^^^^^^^^^^^^^^^ required by this bound in `Handler::handle::{anon_assoc#0}` + | ) -> impl Future + Send; + | ^^^^^^^^^^^^^^^^^^^ required by this bound in `Handler::handle::{anon_assoc#0}` diff --git a/concurrency/tests/ui/actor_two_handler_attributes.stderr b/concurrency/tests/ui/actor_two_handler_attributes.stderr index 972dc80ca..e00130122 100644 --- a/concurrency/tests/ui/actor_two_handler_attributes.stderr +++ b/concurrency/tests/ui/actor_two_handler_attributes.stderr @@ -1,4 +1,4 @@ -error: a method can carry only one of #[handler], #[send_handler] and #[request_handler] +error: a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler] --> tests/ui/actor_two_handler_attributes.rs:14:5 | 14 | #[request_handler] diff --git a/concurrency/tests/ui/actor_unknown_param.stderr b/concurrency/tests/ui/actor_unknown_param.stderr index 3d33b4cd2..fe0d8cc0b 100644 --- a/concurrency/tests/ui/actor_unknown_param.stderr +++ b/concurrency/tests/ui/actor_unknown_param.stderr @@ -1,4 +1,4 @@ -error: unknown parameter, expected `protocol`, `allow_shared_state` or `allow_panic_abort` +error: unknown parameter, expected `protocol`, `max_outstanding`, `max_native_jobs`, `allow_shared_state` or `allow_panic_abort` --> tests/ui/actor_unknown_param.rs:11:9 | 11 | #[actor(protocols = CounterProtocol)] diff --git a/concurrency/tests/ui/deferred_handler_return_type.rs b/concurrency/tests/ui/deferred_handler_return_type.rs new file mode 100644 index 000000000..6d00947e8 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_return_type.rs @@ -0,0 +1,17 @@ +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, PendingReply, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter(u64); +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) -> u64 { + self.0 + } +} +fn main() {} diff --git a/concurrency/tests/ui/deferred_handler_return_type.stderr b/concurrency/tests/ui/deferred_handler_return_type.stderr new file mode 100644 index 000000000..8fa1ddc06 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_return_type.stderr @@ -0,0 +1,5 @@ +error: #[deferred_handler] answers through its PendingReply; remove the return type + --> tests/ui/deferred_handler_return_type.rs:13:91 + | +13 | fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) -> u64 { + | ^^^ diff --git a/concurrency/tests/ui/deferred_handler_without_reply.rs b/concurrency/tests/ui/deferred_handler_without_reply.rs new file mode 100644 index 000000000..b5f2a618f --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_without_reply.rs @@ -0,0 +1,15 @@ +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter(u64); +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, _ctx: &Context) {} +} +fn main() {} diff --git a/concurrency/tests/ui/deferred_handler_without_reply.stderr b/concurrency/tests/ui/deferred_handler_without_reply.stderr new file mode 100644 index 000000000..10661bc20 --- /dev/null +++ b/concurrency/tests/ui/deferred_handler_without_reply.stderr @@ -0,0 +1,5 @@ +error: #[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found 3 + --> tests/ui/deferred_handler_without_reply.rs:13:5 + | +13 | fn read(&mut self, _msg: Read, _ctx: &Context) {} + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/concurrency/tests/ui/pending_reply_stored.rs b/concurrency/tests/ui/pending_reply_stored.rs new file mode 100644 index 000000000..253a661e7 --- /dev/null +++ b/concurrency/tests/ui/pending_reply_stored.rs @@ -0,0 +1,21 @@ +// A `PendingReply` is turn-scoped: storing it without `defer()` (which takes +// a credit) does not compile. +#![allow(unused_imports)] +use spawned_concurrency::{actor, protocol, PendingReply, Response}; +#[protocol] +pub trait CounterProtocol: Send + Sync { + fn read(&self) -> Response; +} +use counter_protocol::Read; +pub struct Counter { + held: Vec>, +} +use spawned_concurrency::threads::{Actor, Context, Handler}; +#[actor] +impl Counter { + #[deferred_handler] + fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) { + self.held.push(reply); + } +} +fn main() {} diff --git a/concurrency/tests/ui/pending_reply_stored.stderr b/concurrency/tests/ui/pending_reply_stored.stderr new file mode 100644 index 000000000..680ecee04 --- /dev/null +++ b/concurrency/tests/ui/pending_reply_stored.stderr @@ -0,0 +1,17 @@ +error[E0521]: borrowed data escapes outside of method + --> tests/ui/pending_reply_stored.rs:18:9 + | +17 | fn read(&mut self, _msg: Read, reply: PendingReply<'_, u64>, _ctx: &Context) { + | ----- + | | + | `reply` is a reference that is only valid in the method body + | has type `PendingReply<'1, u64>` +18 | self.held.push(reply); + | ^^^^^^^^^^^^^^^^^^^^^ + | | + | `reply` escapes the method body here + | argument requires that `'1` must outlive `'static` + | + = note: requirement occurs because of a mutable reference to `Vec>` + = note: mutable references are invariant over their type parameter + = help: see for more information about variance diff --git a/concurrency/tests/ui/reply_to_not_clone.rs b/concurrency/tests/ui/reply_to_not_clone.rs new file mode 100644 index 000000000..3bc994b47 --- /dev/null +++ b/concurrency/tests/ui/reply_to_not_clone.rs @@ -0,0 +1,6 @@ +// A `ReplyTo` answers exactly once: it is not `Clone`. +use spawned_concurrency::ReplyTo; +fn duplicate(token: &ReplyTo) -> ReplyTo { + token.clone() +} +fn main() {} diff --git a/concurrency/tests/ui/reply_to_not_clone.stderr b/concurrency/tests/ui/reply_to_not_clone.stderr new file mode 100644 index 000000000..d361802f6 --- /dev/null +++ b/concurrency/tests/ui/reply_to_not_clone.stderr @@ -0,0 +1,15 @@ +error[E0308]: mismatched types + --> tests/ui/reply_to_not_clone.rs:4:5 + | +3 | fn duplicate(token: &ReplyTo) -> ReplyTo { + | ------------ expected `ReplyTo` because of return type +4 | token.clone() + | ^^^^^^^^^^^^^ expected `ReplyTo`, found `&ReplyTo` + | + = note: expected struct `ReplyTo` + found reference `&ReplyTo` +note: `ReplyTo` does not implement `Clone`, so `&ReplyTo` was cloned instead + --> tests/ui/reply_to_not_clone.rs:4:5 + | +4 | token.clone() + | ^^^^^ diff --git a/deny.toml b/deny.toml index e6a8076ec..44baa3dae 100644 --- a/deny.toml +++ b/deny.toml @@ -19,6 +19,11 @@ ignore = [ # fxhash (unmaintained) via monoio 0.2, only in the private `io-experiments` # benchmark feature. No replacement release of monoio drops it yet. { id = "RUSTSEC-2025-0057", reason = "fxhash via monoio; private io-experiments feature only" }, + # pin-utils (unmaintained, no vulnerability) via loom 0.7's `futures` + # feature, a dev-dependency only under `--cfg spawned_scheduler_loom` (the + # reply-slot Loom model drives a future with `loom::future::block_on`). + # Never part of a shipped build. Drop when loom stops depending on it. + { id = "RUSTSEC-2026-0441", reason = "pin-utils via loom's futures feature; Loom model-check dev-dependency only" }, ] [licenses] diff --git a/docs/GUIDE.md b/docs/GUIDE.md index ce2c54860..0d9708aaf 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -20,13 +20,15 @@ or model that checks each one, are collected in the - [Type Erasure: Recipient and Receiver](#type-erasure-recipient-and-receiver) - [Registry](#registry) - [Response\](#responset) +- [Deferred replies and pipe-to-self](#deferred-replies-and-pipe-to-self) (v0.8) - [Message Trait](#message-trait) - [Error Handling](#error-handling) - [spawned-rt](#spawned-rt) -- [Links and trapped exits](#links-and-trapped-exits) +- [Links, monitors and exit signals](#links-monitors-and-exit-signals) (v0.8) - [Execution groups and supervision](#execution-groups-and-supervision) (v0.7) - [Metrics](#metrics) - [Testing your actors](#testing-your-actors) +- [Experimental in 0.8](#experimental-in-08) - [Further reading](#further-reading) --- @@ -37,7 +39,7 @@ Every actor goes through three phases: 1. **`started()`** โ€” called once before the actor begins processing messages. Use this for initialization: starting timers, registering with the registry, etc. 2. **Message loop** โ€” the actor processes messages one at a time via `Handler` implementations. -3. **`stopped()`** โ€” called once after the message loop exits. Use this for cleanup. +3. **`stopped()`** โ€” called once after the message loop exits, unless the exit was a kill or an exit signal (see [When `stopped()` does not run](#when-stopped-does-not-run)). Use this for cleanup. ```rust #[actor(protocol = MyProtocol)] @@ -66,9 +68,9 @@ All three phases are wrapped in `catch_unwind`: - **Panic in `started()`** โ€” the actor stops immediately. No messages are processed, `stopped()` is not called. - **Panic in a handler** โ€” the current message is lost. The actor stops and `stopped()` is called. -- **Panic in `stopped()`** โ€” the panic is logged and `join()` still returns, but the exit reason becomes `Panic("panic in stopped(): ...")` unless it was already abnormal. +- **Panic in `stopped()`** โ€” the panic is logged and `join()` still returns, but the exit reason becomes `Panic("panic in stopped(): ...")` unless the actor was killed (a kill's `Killed` stands). -In all cases, `join()` will eventually return and subsequent `send()`/`request()` calls return `Err(ActorStopped)`. +In all cases, `join()` will eventually return and subsequent `send()`/`request()` calls return `Err(NoProc)`. ### Compile-time lints @@ -108,9 +110,42 @@ The flags combine with protocols: `#[actor(protocol = P, allow_shared_state)]`. Call `ctx.stop()` from inside a handler or lifecycle hook. This cancels the actor's internal token, causing the message loop to exit after the current handler finishes. -From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop()` exits `Normal`, `shutdown()` exits `Shutdown`, and `stop_with_reason(reason)` picks the reason. All are cooperative: the running handler finishes and `stopped()` runs. They ignore `trap_exit`. `ActorRef` itself has no stop method; an explicit shutdown message in your protocol also works. +From outside, use the actor's `ChildHandle` (`actor_ref.child_handle()`): `stop()` exits `Normal` and `stop_with_reason(reason)` picks the reason (`stop_with_reason(ExitReason::Shutdown)` for a shutdown). All are cooperative: the running handler finishes and `stopped()` runs. They ignore `trap_exit`. `ActorRef` itself has no stop method; an explicit shutdown message in your protocol also works. -Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped`. +Stopping discards messages still queued: once the stop is observed no further message is handled, and queued requests fail with `ActorStopped(reason)`, the actor's final exit reason. + +### When `stopped()` does not run + +As in OTP (`terminate/2` runs only for exits the process takes itself), `stopped()` runs for `ctx.stop()`, an administrative stop (`ChildHandle::stop`/`shutdown`/`stop_with_reason`), a panic in a handler, and a parent's exit to a **trapping** actor. It is **skipped** when the exit is begun by: + +- a kill (`ChildHandle::kill()`, `exit(Kill)`, a supervisor's kill escalation at `shutdown_timeout`), unless `stopped()` already began; +- an untrapped exit signal or link notice (`ChildHandle::exit(reason)`, a linked peer's abnormal exit); +- a supervisor's or system's shutdown, or a `start_linked` parent's exit, to an actor that does **not** trap exits: the parent's exit is an exit signal (design E3); +- the runtime dropping the actor (`Killed`). + +Every skip is logged at `debug` with `event = "stopped_skipped"`, the actor id and type, `cause` (`kill`, `exit_signal` or `runtime_drop`) and `reason`. The log is emitted whether or not the actor overrides `stopped()` (there is no cheap way to tell), so filter on the actor type. The lifecycle trace (feature `lifecycle-trace`) records the same decision as `Hook` `skip`. + +If cleanup must run on a supervisor's shutdown, trap exits first thing in `started()`; the parent's exit then ends the actor through `stopped()`, after the messages queued before it: + +```rust +async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); // shutdown runs stopped(); linked peers' deaths arrive as `Exit` +} +``` + +A trapping actor still handles the messages queued before the parent's exit. To refuse late work, check `ctx.exit_pending()`: it is `true` once the actor's exit has begun (`ctx.stop()`, an administrative stop, a kill, an untrapped signal, a panic; also in `stopped()`) or a parent exit is queued behind the current message. A trapped `Exit` from a linked peer or a non-parent `exit` is not a pending exit. It is advisory: a signal can arrive right after it returns `false`. + +```rust +#[request_handler] +async fn handle_work(&mut self, msg: Work, ctx: &Context) -> Result<(), Refused> { + if ctx.exit_pending() { + return Err(Refused::ShuttingDown); + } + // ... +} +``` + +State that must be released on every exit, including kills and runtime drops, belongs in `Drop`. --- @@ -124,20 +159,23 @@ Stopping discards messages still queued: once the stop is observed no further me | `ctx.send(msg)` | Send a fire-and-forget message to this actor (self-send) | | `ctx.request(msg)` | Send a request and wait for the reply (tasks: async, threads: blocking) | | `ctx.request_with_timeout(msg, duration)` | Send a request with a custom timeout | -| `ctx.request_raw(msg)` | Send a request and get a raw oneshot receiver | +| `ctx.send_request(msg, deadline)` | Send a request without waiting (`gen_server:send_request/2` with an explicit deadline) and get its `ReplyReceiver` (tasks: a future, threads: `recv()`/`try_recv()`) yielding `Result`; the deadline (a `Duration`, an `Instant` or `Deadline::Never`) is fixed at send | | `ctx.recipient::()` | Get a type-erased `Recipient` for this actor | | `ctx.actor_ref()` | Get an `ActorRef` from the context | | `ctx.id()` | This actor's `ActorId` | | `ctx.monitor(&child_handle)` | Receive a `Down` message when the target exits; returns a `MonitorRef` | -| `ctx.demonitor(monitor_ref)` | Cancel a monitor | -| `ctx.link(&child_handle)` / `ctx.unlink(&child_handle)` | Bidirectional exit propagation (see [Links and trapped exits](#links-and-trapped-exits)) | -| `ctx.trap_exit(bool)` | Receive linked exits as messages instead of being cancelled | +| `ctx.demonitor(monitor_ref)` | Cancel a monitor (a fence with OTP's `flush`: no `Down` afterwards, not even one already queued) | +| `ctx.link(&child_handle)` / `ctx.unlink(&child_handle)` | Bidirectional exit propagation (see [Links, monitors and exit signals](#links-monitors-and-exit-signals)) | +| `ctx.try_link(&child_handle)` | Link, or `Err(NoProc)` if the target is already gone | +| `ctx.exit(&child_handle, reason)` | Send an exit signal from this actor (OTP `exit/2`) | +| `ctx.trap_exit(bool)` | Receive exit signals and link notices as `Exit` messages instead of exiting | +| `ctx.exit_pending()` | Whether this actor's exit has begun or a parent exit is queued | | `ctx.spawn_helper(future)` / `ctx.spawn_blocking_helper(f)` | Actor-owned helper tasks, cancelled and joined before cleanup completes (v0.7) | | `ctx.register_name(name)` | Register this actor under a name, released with its generation (v0.7) | | `ctx.execution_group()` | The `ExecutionGroup` a managed actor runs on (v0.7) | | `ctx.mailbox_depth()` | Queued messages (with the `metrics` feature) | -`Context::from_ref(&actor_ref)` creates a context from an `ActorRef`, useful for setting up timers or stream listeners from outside the actor. +`actor_ref.context()` returns the actor's `Context`, useful for setting up timers or stream listeners from outside the actor. ### Self-scheduling example @@ -162,7 +200,7 @@ async fn handle_tick(&mut self, _msg: Tick, ctx: &Context) { | `actor_ref.request(msg)` | Request with default 5s timeout. Tasks: `.await`, Threads: blocking | | `actor_ref.request_with_timeout(msg, duration)` | Request with custom timeout | | `actor_ref.with_request_timeout(duration)` | Clone a configured handle for generated protocol methods (v0.7) | -| `actor_ref.request_raw(msg)` | Returns a raw oneshot receiver | +| `actor_ref.send_request(msg, deadline)` | Returns the request's `ReplyReceiver` (as `ctx.send_request`) | | `actor_ref.recipient::()` | Get a type-erased `Recipient` | | `actor_ref.context()` | Get a `Context` (for timer setup, etc.) | | `actor_ref.join()` | Wait until the actor has fully stopped (tasks: `.await`, threads: blocking) | @@ -409,7 +447,7 @@ When you need to send a specific message type to an actor without knowing its co pub type Recipient = Arc>; ``` -`Receiver` is the object-safe trait that provides `send()` and `request_raw()` for a single message type. +`Receiver` is the object-safe trait that provides `send()` and `send_request()` for a single message type (on a `Recipient`, `send_request` takes a `Deadline`: `Deadline::after(d)` or `d.into()`). It is sealed: `ActorRef` and `Context` implement it, and test doubles come from `spawned_test`. ### Getting a Recipient @@ -479,19 +517,28 @@ The registry uses `Any`-based downcasting, so `whereis` returns `None` if the st `Response` is the return type for protocol request methods. It works in both execution modes: -- **Tasks mode** โ€” wraps a oneshot receiver. Use `.await` to get `Result`. -- **Threads mode** โ€” wraps a pre-computed result. Use `.unwrap()` or `.expect()` directly. +- **Tasks mode** โ€” wraps the request's `ReplyReceiver`. Use `.await` to get `Result`. +- **Threads mode** โ€” wraps a pre-computed result. Use `.unwrap()`, `.expect()` or `.into_result()` directly. -### Methods (sync, for threads mode) +### Methods (sync) | Method | Description | |--------|-------------| +| `.into_result()` | The `Result` | | `.unwrap()` | Extract the value, panic on error | | `.expect(msg)` | Extract the value, panic with custom message on error | -| `.is_ok()` | Returns `true` if the response contains `Ok` | -| `.is_err()` | Returns `true` if the response contains `Err` | +| `.is_ok()` | Returns `true` if the response is ready and `Ok` | +| `.is_err()` | Returns `true` if the response is ready and `Err` | | `.map(f)` | Transform the inner value if `Ok` | +A tasks-mode response that was not awaited is *pending*. Off an async +runtime (plain synchronous code), `into_result`, `unwrap`, `expect` and `map` +block until the reply, its deadline or the actor's death. Inside a runtime +they panic, since blocking an executor thread can deadlock the actor that +must answer: `.await` it. `is_ok` and `is_err` borrow and cannot wait, so on +a pending response they panic instead of answering `false` (0.7 answered +`false`, and `unwrap` always panicked). + ### Async usage (tasks mode) ```rust @@ -542,15 +589,223 @@ before message arguments move. Send-only methods have no timeout, and a `request_timeout` attribute on one is a compile error. Manual message calls use `request_with_timeout(msg, limit)`. -Enqueue still happens when the generated method is called. In async mode the -response timer starts when its future is first polled. In native mode the -protocol call blocks immediately. Timeout/drop does not cancel accepted work or +Enqueue still happens when the generated method is called. The deadline is +fixed then, at send (send time plus the timeout, as OTP's `call`), and travels +with the request; in async mode the response future waits until that deadline +when polled. In native mode the protocol call blocks immediately. Timeout/drop does not cancel accepted work or roll back side effects; decide retry safety in application code. These settings do not propagate deadlines to downstream requests. Run the [asserted example](../examples/capabilities/src/bin/protocol_timeouts.rs) in both modes, or follow the [0.7 client migration guide](MIGRATION.md#preparing-for-07). +## Deferred replies and pipe-to-self + +A handler that would await downstream work blocks its whole mailbox. Instead, +take the request's reply obligation out of the turn and answer it later; the +caller and the protocol do not change. + +Every `Handler::handle` receives a `PendingReply<'_, R>`. `#[request_handler]` +methods answer it for you with their return value. A `#[deferred_handler]` +receives it: + +```rust +#[actor(protocol = PeerTableProtocol)] +impl PeerTable { + #[deferred_handler] + async fn wait_for_peer(&mut self, msg: WaitForPeer, reply: PendingReply<'_, Lease>, + _ctx: &Context) { + if let Some(lease) = self.take_lease(&msg.caps) { + reply.reply(lease); + return; + } + match reply.defer() { + Ok(token) => self.waiters.push_back((msg.caps, token)), // a ReplyTo + Err(DeferError::Overloaded) => {} // the caller was already answered Overloaded + Err(DeferError::Closed) => {} // the caller timed out or left + Err(DeferError::NoCaller) => {} // reached by `send` + Err(_) => {} // `DeferError` is #[non_exhaustive] + } + } +} +``` + +Rules (the [design](design/deferred-replies.md) has the full contract): + +- `defer()` takes one of the instance's `Actor::MAX_OUTSTANDING` credits + (default 1024). At the bound the caller gets `Err(Overloaded)` at once. +- A `ReplyTo` is `Send + 'static` and answers exactly once: + `token.reply(v)` returns a `Delivery` (`Delivered`, `CallerGone`, `Expired`, + `OwnerExited`); `token.overloaded()` answers `Err(Overloaded)`. As in OTP, + the caller's timeout is the only deadline authority: a reply reads no + clock, so `Expired` means the caller's expiry already ran (it sees + `RequestTimeout`); a reply just after the deadline, before that expiry, + is still `Delivered` and the caller gets the value. +- Dropping an unanswered token answers `ReplyDropped`. If the turn panicked or + the actor is exiting, it is held instead and answered at the actor's death + with its final reason. `stopped()` may still answer stored tokens; at the + actor's death every open token answers `ActorStopped(reason)`. +- The deadline is the caller's, fixed at send. `token.is_closed()` tells you + the caller can no longer be answered; the actor's deadline heap expires + stored tokens on time and returns their credits. +- `PendingReplies` is a bounded table of tokens by key (purges terminal + entries on insert; `InsertError::{Duplicate, Full}` hands the token back; `into_parts()` + gives the `InsertErrorKind` and the token together, so even a reason + added later can be answered). + +**Answering with a reason.** The framework's errors (`ReplyDropped`, +`ActorStopped`, `Overloaded`, `RequestTimeout`) say what happened to the +request, not why your actor could not serve it. To carry your own reason, make +the reply type a `Result`, as `gen_server` replies `{error, Reason}`: + +```rust +impl Message for GetPeer { type Result = Result; } + +// a ReplyTo> answers either way, exactly once: +token.reply(Err(PeerError::Disconnected)); +// the caller sees Ok(Err(PeerError::Disconnected)): the request worked, +// the answer is "no"; Err(ActorError::..) stays the transport's. +``` + +**Send-only handlers.** A manual `impl Handler` for a message whose +`Result` is `()` can leave out the reply parameter with `#[send_handler]` on +the impl; the impl answers `()` once `handle` returns (also after an early +`return`), so a `request` gets `Ok(())` rather than `ReplyDropped`. This is +the manual form of a `#[send_handler]` method in an `#[actor]` block: + +```rust +#[send_handler] +impl Handler for Poller { + async fn handle(&mut self, _msg: Tick, ctx: &Context) { + self.poll().await; + } +} +``` + +**Pipe-to-self jobs** run work owned by the actor and deliver the result as a +message in a later turn (OTP's `Task.Supervisor.async_nolink` plus +`handle_info`). A gossip verdict checked off the actor, with a deadline, that +reports exactly one outcome per object even when the actor is at a bound: + +```rust +#[send_handler] +async fn on_gossip(&mut self, msg: Gossip, ctx: &Context) { + // Reserve first: on refusal nothing has been moved into the job yet. + let opts = JobOptions::new().deadline(VERDICT_DEADLINE); + let Ok(permit) = ctx.reserve_blocking_job(opts) else { + return self.report(msg.id, Verdict::Ignore(Overloaded)); + }; + let Gossip { id, object } = msg; + permit.pipe_blocking(move |_job| check(&object), move |res| Checked { id, res }); +} + +#[send_handler] +async fn on_checked(&mut self, msg: Checked, _ctx: &Context) { + let verdict = match msg.res { + Ok(outcome) => outcome, + Err(JobError::DeadlineExceeded) => Verdict::Ignore(TimedOut), + Err(_) => Verdict::Reject(CheckFailed), // a panic in `check` + }; + self.report(msg.id, verdict); +} + +// Delegation: the obligation and its credit stay here. +let token = reply.defer()?; +let downstream = other.send_request(Lookup(key), token.deadline()); // same deadline as our caller +ctx.pipe_reply(JobOptions::new(), token, |_| async move { downstream?.await }, |res, token| Delegated { res, token })?; +``` + +- Four calls, one per shape: `ctx.pipe` (async work), `ctx.pipe_blocking`, + `ctx.pipe_reply` and `ctx.pipe_reply_blocking` (linked to a deferred + reply). Each takes `JobOptions` first, built with `JobOptions::new()` and + `.key(k)`, `.deadline(d)` (a `Duration`, `Instant` or `Deadline`), + `.emit_credits(n)` and, for `pipe_reply*` only, + `.cancel_on_caller_gone()`. The work receives the job's `JobCtx` by + value (`|_| fut` when it needs neither cancellation nor emits). +- For async jobs (`pipe`, `pipe_reply`), `work` is a factory that runs at + once, in the current turn: only the future it returns is the job. A panic + before its `async` block is a handler panic, not `JobError::Panicked`, + and a job the owner's exit refuses meanwhile has already run its factory. + Keep preparation inside the `async` block. Blocking jobs run their whole + closure on the job's thread, under the job's panic boundary. +- `done(outcome)` runs in the owner's turn. `JobOutcome` is + `Result`; `Err` carries `JobError::Panicked` or + `JobError::DeadlineExceeded`. `JobError` and `PipeError` are + `#[non_exhaustive]`: match with a wildcard arm. +- `ctx.pipe*` reject at once at a bound (`PipeError`, never waiting) and drop + `work` and `done`. When the caller must still produce a result, + `ctx.reserve_job(opts)` / `ctx.reserve_blocking_job(opts)` take the + admission first (owner running, a credit, and for blocking jobs a native + slot) and return a permit whose `pipe` / `pipe_blocking` cannot fail. + `pipe_reply*` errors hand the token back. +- `JobOptions::key` cancels the previous job with that key and + fences its queued messages; `JobHandle::cancel` fences one job. +- `JobCtx::emit_blocking` (and `emit` for async jobs) sends fenced messages, + bounded by `JobOptions::emit_credits`; the terminal message is always last. + `JobCtx::emit_at(at, msg)` (`emit_at_blocking` in blocking jobs) delivers + at `at`, fenced and credit-accounted the same way: it never reaches a newer + job's turn or a newer instance, it is dropped if the job is cancelled or + replaced first, and the job's terminal message waits for it. An `at` + already reached, or in the past, is an immediate emit (enqueued at the + call, ahead of later emits): no need to branch to `emit` yourself. An + `at` past `JobOptions::deadline` is accepted, not an error: if the + deadline passes first it takes the notification and the emit is dropped + at its instant (if the job finishes first, the emit and then the result + arrive as usual). +- **Job deadlines are hard.** When `JobOptions::deadline` passes, `done` gets `DeadlineExceeded` at once, later + emits fail with `JobGone` and the work's eventual result is dropped. The + work itself keeps running (cooperative cancellation) and keeps its credit + and native slot until it returns. For a **soft deadline** (stop starting + new work, finish and deliver what is in hand), leave `JobOptions::deadline` + unset and check the time in the work: + + ```rust + let stop_starting_at = Instant::now() + SOFT_DEADLINE; + ctx.pipe_blocking(JobOptions::new(), move |job| { + let mut done = Vec::new(); + for item in batch { + if job.is_cancelled() || Instant::now() >= stop_starting_at { break; } + let out = prove(item); // a started item finishes + let _ = job.emit_blocking(Proven(out.id)); // and is delivered + done.push(out); + } + Summary::from(done) + }, |res| BatchDone { res })?; + ``` + + `spawned_concurrency::Instant` (also `tasks::Instant`, `threads::Instant`) + is the deadline clock: `std::time::Instant`, or the simulated clock under + `--cfg spawned_sim`, so code need not depend on `spawned-rt`. +- **Bounds.** Each unlinked job holds one `MAX_OUTSTANDING` credit; blocking + jobs also count against `Actor::MAX_NATIVE_JOBS` (default + `min(4, available_parallelism)`). + + > **Size the native budget.** The default of at most 4 blocking jobs per + > actor is easy to outgrow now that blocking jobs run on Tokio's blocking + > pool: an actor that verifies, proves or does I/O in blocking jobs + > usually needs tens or hundreds (one client port needed 352). Under load + > the only symptom is `PipeError::NativeBudget` refusals; the first one + > per actor instance is logged as a `native_budget_exhausted` warning. + + Set both in the macro, + `#[actor(max_native_jobs = 64, max_outstanding = 4096)]` (or as `Actor` + constants in a manual impl), and override them per supervised child with + `ChildSpec::max_native_jobs(n)` / `ChildSpec::max_outstanding(n)`. +- **Executors.** In `tasks` mode async jobs run on the owner's runtime and + blocking jobs on Tokio's blocking pool (`spawn_blocking` of the actor's + group runtime, or the current runtime), still bounded by the native + budget; a runtime that is dropped waits for blocking jobs still running, + as with any `spawn_blocking`. `threads` mode offers `pipe_blocking` + and `pipe_reply_blocking` on owned native threads (a reservation + spawns the thread, parked until the job starts). +- Owner exit cancels every job; after death the jobs are joined, bounded by + the cleanup deadline, and stragglers are reported (`unfinished_at_stop`). + +Test a manual handler directly with `spawned_test::reply_pair()`, and run +code inside a live actor's turn (to start a job or inspect state) with +`spawned_test::tasks::in_actor` / `spawned_test::threads::in_actor`. Run the +[deferred replies example](../examples/capabilities/src/bin/deferred_replies.rs). + ## Message Trait The `Message` trait defines a message type and its expected reply type: @@ -582,16 +837,32 @@ impl Message for GetCount { All actor communication can fail with `ActorError`: ```rust +#[non_exhaustive] pub enum ActorError { - ActorStopped, // The actor has stopped or its mailbox is closed - RequestTimeout, // A request exceeded the timeout (default 5s) + ActorStopped(ExitReason), // The actor died before replying, with this final reason + NoProc, // The actor was already gone when sent to (OTP noproc) + RequestTimeout, // A request exceeded its deadline (default 5s) + ReplyDropped, // The handler dropped the request unanswered + Overloaded, // The actor rejected the request at a bound } ``` -- `send()` returns `Err(ActorStopped)` once the actor's mailbox has closed. While the actor is stopping (after a stop request, including during `stopped()`), `send()` still returns `Ok(())` and the message is dropped without being handled: `Ok` means accepted, not handled. -- `request()` returns `Err(ActorStopped)` if the actor stops before replying, or `Err(RequestTimeout)` if the reply doesn't arrive in time. +- `send()` returns `Err(NoProc)` once the actor is past its death point. While the actor is stopping (after a stop request, including during `stopped()`), `send()` still returns `Ok(())` and the message is dropped without being handled: `Ok` means accepted, not handled. +- `request()` returns `Err(ActorStopped(reason))` if the actor dies before replying, with the reason `wait_exit()` and monitors report (`Killed`, `Panic(..)`, the exit signal's reason, ...), `Err(NoProc)` if it was already gone, or `Err(RequestTimeout)` if the reply doesn't arrive in time. A request whose handler panicked is answered when the actor dies (after `stopped()`), not at once. - `request_with_timeout()` lets you specify a custom timeout. +`ActorError` is `#[non_exhaustive]`. Classify it with the helpers instead of an exhaustive match: + +| Variant | `target_gone()` | `is_timeout()` | `is_transient()` | `exit_reason()` | +|---|---|---|---|---| +| `ActorStopped(r)` | `true` | `false` | `false` | `Some(&r)` | +| `NoProc` | `true` | `false` | `false` | `None` | +| `RequestTimeout` | `false` | `true` | `true` | `None` | +| `ReplyDropped` | `false` | `false` | `false` | `None` | +| `Overloaded` | `false` | `false` | `true` | `None` | + +`target_gone()`: retrying the same reference cannot succeed (look the actor up again; a supervisor may have restarted it). `is_transient()`: retrying the same target may succeed. `ReplyDropped` is neither: the actor is alive but its handler chose not to answer (a handler that means "not now, try again" answers `reply.overloaded()`, which is transient; a caller whose protocol retries dropped requests matches `ReplyDropped` explicitly). `ExitReason::is_abnormal()` is `false` only for `Normal` and `Shutdown` (`Killed` and `NoProc` are abnormal). + --- ## spawned-rt @@ -643,55 +914,148 @@ Wraps standard library primitives: See [Execution Modes and Backends](#execution-modes-and-backends) for choosing between `tasks` and `threads`. + -## Links and trapped exits +## Links, monitors and exit signals + +Exit signals follow Erlang/OTP (v0.8, [design](design/otp-exit-signals.md)). +Everything in this section works in both modes and across modes; use the +`threads` imports and synchronous hooks in synchronous mode. The +[exit_signals example](../examples/capabilities/src/bin/exit_signals.rs) runs +each rule below. -Available since v0.6.0. +### Exit signals -Links are bidirectional failure notifications. Use `ctx.link(&child_handle)` and -`ctx.unlink(&child_handle)` for existing actors, or `child.start_linked(&parent_ctx)` -to perform registration before the child starts. Child handles allow links across -execution modes. Normal peer exits are ignored by non-trapping actors; abnormal -exits request cooperative cancellation and propagate their cause. +`ChildHandle::exit(reason)` sends an exit signal from outside any actor +(`from` is `ActorId::EXTERNAL`); `ctx.exit(&target, reason)` sends it from +the calling actor; `ChildHandle::kill()` is `exit(ExitReason::Kill)`. The +target applies OTP's `exit_signal/2` rules, with the trap flag read when the +signal arrives: -Call `ctx.trap_exit(true)` to receive linked exits through `Actor::exit_received` -instead. The actor macro supports this lifecycle hook: +| Signal | Non-trapping target | Trapping target | +|---|---|---| +| `Kill` | ends `Killed` | ends `Killed` (untrappable) | +| `Normal` | ignored | `Exit { from, reason: Normal }` queued | +| any other reason | ends with that reason after its current handler, without `stopped()`; its queue is discarded | `Exit { from, reason }` queued | +| from the target's parent | as above | a parent exit is queued (see [Parent exits](#parent-exits)) | + +A kill supersedes any other reason until the actor's death point: an actor +already stopping (even inside `stopped()`) ends `Killed`, and a panic after +the kill does not replace it. `stopped()` is skipped if it has not begun; a +`stopped()` that is running finishes, since Rust cannot interrupt it. A +non-kill signal to a non-trapping actor that is already exiting replaces its +reason, unless it was killed; a signal after the death point does nothing. +Signals that arrive while `started()` runs take effect when it returns, with +the trap flag read then, so `ctx.trap_exit(true)` first thing in `started()` +covers them. + +`stop()` and `stop_with_reason(r)` (on a `ChildHandle` or the +`Context`) are not exit signals. They are administrative stops, as +`gen_server:stop`: untrappable, `stopped()` runs, and the first stop's reason +wins over later stops. `stop_with_reason(ExitReason::Kill)` is such a stop +and ends `Kill`; only a `Kill` exit signal (`kill()`, `exit(Kill)`) +produces `Killed`. + +### Links and trapped exits + +Links are bidirectional. Use `ctx.link(&child_handle)` and +`ctx.unlink(&child_handle)` for existing actors, or +`child.start_linked(&parent_ctx)`, which links before the child's `started()` +and makes the caller its parent. Repeated links and links to oneself are +no-ops. When a linked actor dies, each peer receives a link notice with its +final reason, under the same rules as an exit signal except that a `Kill` +notice is trappable: a non-trapping peer ignores `Normal` and otherwise ends +with the same reason, without `stopped()`, and propagates it to its own links. + +Call `ctx.trap_exit(true)` to receive signals and notices through +`Actor::exit_received` instead: ```rust use spawned_concurrency::{actor, Exit}; use spawned_concurrency::tasks::{Actor, Context}; -struct Supervisor; +struct Watcher; #[actor] -impl Supervisor { +impl Watcher { #[started] async fn setup(&mut self, ctx: &Context) { ctx.trap_exit(true); } #[exit_received] - async fn child_exit(&mut self, exit: Exit, _ctx: &Context) { + async fn peer_exit(&mut self, exit: Exit, _ctx: &Context) { tracing::warn!(actor = %exit.from, reason = ?exit.reason, "Linked actor exited"); } } ``` -Use synchronous methods and the `threads` imports in synchronous mode. The hook -is optional and defaults to doing nothing. Trapped exits use the ordinary actor -mailbox; they do not interrupt a running handler. A direct -`stop_with_reason(ExitReason::Kill)` bypasses trapping and requests cooperative -cancellation. Propagated Kill notices are trappable, so they cannot bypass -another actor's failure boundary. The notice keeps the existing Kill enum value. A normal stop already committed is -not relabelled by a later peer failure; callback failures are still reported. - -A link to a closed peer creates no retained relationship and applies that peer's -exit-signal rules for the attempt. Repeated live links are no-ops. Unlink prevents -future propagation through the relationship but does not retract an exit already -in flight. Links are a supervision primitive. Managed OneForOne/OneForAll/RestForOne recovery is described in -[execution groups and supervision](#execution-groups-and-supervision). -Transparent reference rebinding remains separate work. The [review and regression evidence](https://github.com/lambdaclass/spawned/blob/efba24340a01df82f872ec422b9dbaf48f897b04/legacy/docs/reviews/pr-166-links.md) is archived. +The hook is optional and defaults to doing nothing. `Down` carries the +monitor's `MonitorRef`, the monitored actor's `ActorId` (`actor`, OTP's +`Object`) and its reason; `Exit` carries `from` and `reason`. Both are +`#[non_exhaustive]`: match them with `..`. `Exit`, `Down` and +parent exits are ordinary mailbox messages: they queue behind messages already +there and never interrupt a running handler. A death notice follows every +message the dying actor sent earlier to the same receiver. A link notice is +queued before the dying actor's `wait_exit()` returns; a `Down` is queued by +the monitor shortly after. + +A link or monitor made through a `Context` whose actor is already exiting (for +example from a helper that outlives it) is inert: no record and no notice. + +### Parent exits + +An actor's parent is fixed when it starts: the actor passed to +`start_linked`, the supervisor of a supervised child, or the `ActorSystem` of +a standalone `start_on` actor; a plain `.start()` actor has none. A signal +from the parent to a **trapping** actor is queued as a parent exit: when +dispatched, the actor exits with that reason (`Normal` included) and +`stopped()` runs; `exit_received` never sees it. A non-trapping actor treats +it as any exit signal. + +Supervisor and system shutdown are the parent's `Shutdown` signal. A +non-trapping child ends `Shutdown` without `stopped()`. A trapping child +handles the messages queued before the shutdown, then runs `stopped()` +([`exit_pending()`](#when-stopped-does-not-run) reports the queued shutdown). +A child that has not reached its death point `shutdown_timeout` after its +signal is killed, and reported stuck after a second interval; a system kills +the actors left when its `grace` expires. + +### NoProc + +An actor is gone from its death point on, which comes right after `stopped()` +returns (or is skipped). After it: + +- `ctx.link(&h)` delivers one `Exit { from: h, reason: NoProc }` to a trapping + caller; a non-trapping caller receives that exit signal and ends `NoProc`; +- `ctx.try_link(&h)` returns `Err(NoProc)` and keeps no link; +- `ctx.monitor(&h)` delivers one `Down { reason: NoProc }`; +- `send`/`request` return `Err(ActorError::NoProc)`. + +Before it, including while the target runs `stopped()`, links and monitors +register and fire with the final reason. `ChildHandle::exit_reason()` and +`wait_exit()` give the real reason once it is published, shortly after the +death point. + +### Unlink and demonitor are fences + +Once `ctx.unlink(&h)` returns, that link has no further effect on the caller; +once `ctx.demonitor(r)` returns, no `Down` of that monitor is queued or +handled: one already in the mailbox is dropped unhandled, as OTP's +`demonitor(Ref, [flush])`. An `Exit` already in the mailbox stays. Linking +again creates a new link that gets its own notice. + +### Runtime drop + +An actor whose runtime drops it (runtime shutdown, abort, never polled) ends +`Killed` in both modes, unless a reason other than `Normal` was already +requested. `stopped()` does not run; links receive `Killed` and monitors +`Down { reason: Killed }`. + +Callers see the same final reason: see [Error Handling](#error-handling) +(`ActorStopped(reason)`, `NoProc`). Supervisor startup rules are in +[Construct fresh children after failure](#construct-fresh-children-after-failure). --- @@ -807,10 +1171,12 @@ Use an owned `async_group` for independent scheduling capacity and runtime lifetime guarantees. Ambient `.start()` remains unmanaged, including inside a managed handler; use `ctx.execution_group().start(...)` for owned child work. -The restart limit counts admitted child retry attempts across the supervisor, -including collateral OneForAll/RestForOne replacements, retaining v0.7 behavior. -A six-child OneForAll round needs six available grants. This differs from OTP -intensity accounting per recovery round; it is not changed silently in this patch. +The restart limit counts restart actions across the supervisor, as OTP's +intensity does: a OneForOne child retry is one, and a OneForAll/RestForOne +recovery round is one however many children it rebuilds (a six-child +OneForAll round needs one available grant). The round takes its grant when it +is selected, before the cohort is stopped; a replacement that fails inside the +round requests a new round, which takes another. Retry capacity is checked and reserved before a backoff. An exhausted window fails immediately, even if an older grant would expire during the proposed wait. @@ -859,8 +1225,8 @@ let report = supervisor.shutdown(Duration::from_secs(5)).await; ``` A factory may return an error or panic. Registered construction helpers are -cancelled and joined before retry, including when the factory never returns an -actor. Existing `started()` and `stopped()` hooks keep their signatures. Readiness +cancelled and joined before a replacement is retried, including when the +factory never returns an actor. Existing `started()` and `stopped()` hooks keep their signatures. Readiness commits after `started()` returns and admission remains open; no ordinary handler runs before that barrier. Enqueued startup messages are discarded if startup fails. Calling `ready()` later reports that historical barrier, not current liveness. @@ -872,11 +1238,23 @@ Drop, not only by `stopped()`. Legacy unmanaged `.start()` and synchronous `threads` actors retain their behavior of running both hooks after an early cooperative stop, provided startup itself succeeds. Failed readiness waits for the actual startup outcome, so a non-returning hook -can keep it pending; use a timeout. A startup deadline is a failed attempt and can retry under Permanent/Transient -policy after cleanup, whether it expired in the factory or in `started()`. -A self-requested normal stop before readiness is still a -startup failure: a Transient/Temporary spec fails the tree rather than claiming -a ready service; a Permanent spec can retry within its budget. +can keep it pending; use a timeout. + +The initial start follows OTP's `start_link` (v0.8): + +- A child's **first start** that fails for any reason (factory error or panic, + panic or stop in `started()`, a missed `startup_timeout`, placement or group + closure) fails `start()` whatever the restart mode; it is not retried. Retry + in the application if a dependency may be briefly unavailable at boot. +- The started children roll back in reverse order, and `start()` returns its + error only after every child is terminal (or `Failed` once its cleanup is + stuck): `SupervisorError::FailedToStartChild { child, cause, rollback_failures }`, + or `StartupCleanupStuck`. +- A ready child that exits while later siblings are still starting is + recorded; its exit is handled under the strategy once the whole tree is + ready, as OTP handles a child's `'EXIT'` after `init` returns. +- A replacement that fails before readiness is a failed restart attempt and + is retried under the restart policy and budget. A supervisor accepts only groups owned by its ActorSystem. ActorRefs can still communicate between independent systems. Children start in declaration order @@ -906,7 +1284,7 @@ Supervision emits structured `tracing` events: `restart_scheduled` and at WARN. Fields include `child`, `generation`, `reason` and an available error. Enable an application tracing subscriber to collect them; Spawned installs none. -| Restart mode while supervisor is running | Normal / Shutdown | Panic / Kill | +| Restart mode while supervisor is running | Normal / Shutdown | Panic / Kill / Killed / NoProc | |---|---|---| | Permanent (default) | Restart | Restart | | Transient | Stay stopped | Restart | @@ -914,19 +1292,18 @@ Enable an application tracing subscriber to collect them; Spawned installs none. System/supervisor shutdown suppresses every restart mode. Stopping an individual Permanent child while its supervisor is open intentionally restarts it. Use -supervisor shutdown to close the tree. `ChildHandle::shutdown()` commits Shutdown +supervisor shutdown to close the tree. `stop_with_reason(Shutdown)` commits Shutdown before cooperative cancellation while preserving a previously committed cause; actual handler or cleanup panics remain failures. Existing link propagation still applies and can affect siblings explicitly linked by the application. Receiver-owned cleanup discards queued envelopes individually during completion, -cancellation and unwinding in both modes. An otherwise completed Normal exit -becomes `Panic("queued message destructor: ...")` when a queued destructor fails. -Committed Shutdown, Kill and linked/handler panics keep their primary reason; -`child_handle.mailbox_cleanup_failure()` records the secondary destructor failure -independently. In particular, canceled managed startup remains Shutdown and does -not become restartable because of discarded messages. Runtime-abort fallback -Kill is retained. Every accepted queued reply sender is released before actor -completion, even while ActorRefs remain alive. +cancellation and unwinding in both modes. Queued messages and the actor state +are dropped after the death point, so a panicking destructor does not change +the exit reason; it is recorded in `child_handle.cleanup_failure()`, and a +Transient child whose reason is `Normal` or `Shutdown` restarts when its +cleanup failed. An actor dropped by its runtime ends `Killed`. Every accepted +queued request is answered before actor completion, even while ActorRefs +remain alive. Async sends retain their channel fast path; receiver cleanup closes Tokio's channel and receives through EOF without cooperative-budget deferral. Native @@ -939,9 +1316,10 @@ user value cannot be recovered by the envelope boundary. `ctx.stop_with_reason(ExitReason::Shutdown)` requests a cooperative shutdown; `ExitReason::Kill` requests an abnormal exit without panicking. Both modes finish -the current handler and run `stopped()`. The first committed reason wins, while -an actual cleanup panic can still make the exit abnormal. A transient supervisor -restarts Kill exits but leaves Shutdown exits stopped. `ctx.stop()` keeps Normal. +the current handler and run `stopped()`. The first stop's reason wins over later +stops; a panic, a kill or an untrapped exit signal before the death point still +replaces it. A transient supervisor restarts Kill exits but leaves Shutdown +exits stopped. `ctx.stop()` keeps Normal. The default shared budget permits five retry attempts per rolling 30-second window. Initial construction does not consume a retry; every subsequent attempt @@ -960,6 +1338,19 @@ zero through one second, including 0.15โ€“0.8s. Subsequent caps double up to ready lifetime resets that child's exponent. Cleanup time is not healthy uptime. Use `Backoff::seed` for reproducible policy tests, not security randomness. +`SupervisorConfig` and `Backoff` are `#[non_exhaustive]` (OTP's supervisor +flags grew in OTP 24, and so will these): build them with the setters, + +```rust +let config = SupervisorConfig::new() + .strategy(SupervisorStrategy::OneForAll) + .max_restarts(3) + .restart_window(Duration::from_secs(10)) + .backoff(Backoff::new().initial(Duration::from_millis(50)).jitter(false)); +``` + +or assign the public fields of a `default()` value. + ### Recovery strategies and nested trees Set `SupervisorConfig::strategy` explicitly; OneForOne remains the default. @@ -979,11 +1370,10 @@ notifications are coalesced into a bounded per-child pending set, with one recov cohort at a time. A late old-generation failure is absorbed by its assigned round. A new upstream failure interrupts reconstruction and starts a fresh cleanup round, so dependent factories cannot hold recovery behind a dead upstream. -Initial startup remains serial, retrying a failed construction attempt before -starting dependents. For OneForAll/RestForOne, if an already-ready dependency -exits restartably before the whole tree commits readiness, startup fails and rolls -back the tree. It does not replace that dependency alone underneath initializing -children. A parent supervisor can retry that whole subtree from its factory. +Initial startup is serial and does not retry a failed first start. If an +already-ready dependency exits before the whole tree commits readiness, its +exit is deferred until then and handled under the strategy, so it is not +replaced alone underneath initializing children. The rolling restart budget charges every subsequent factory invocation, including collateral reconstruction and failed constructors. For a large OneForAll tree, @@ -1110,7 +1500,7 @@ Configure before starting actors or constructing observed requests: ```rust use spawned_concurrency::metrics::{self, Config}; -metrics::configure(Config { timing_sample_every: 16 }).unwrap(); +metrics::configure(Config::new().timing_sample_every(16)).unwrap(); // Counters include every operation; one in 16 observations gets timing samples. let snapshot = metrics::snapshot(); let prometheus_text = metrics::render_prometheus(); @@ -1122,7 +1512,7 @@ histogram observations. Configuration is process-wide. The one-time `configure` requirements later, including with a policy previously set by `configure`. For embedded components that initialize independently, use -`metrics::configure_composable(Config { timing_sample_every: 16 })` instead. +`metrics::configure_composable(Config::new().timing_sample_every(16))` instead. It can be called after actors start. The smallest nonzero interval requested by any component wins; zero means โ€œI only require countersโ€ and never disables another component's timings. Requirements last for the process lifetime. Instrumentation initially samples every observation until an explicit policy @@ -1142,6 +1532,8 @@ counters and histograms still aggregate the whole process. | Request outcomes | Started, active, replied, timed out, actor stopped, and caller cancelled. Transport replies are successful even if the application result itself contains an error. | | Lifecycle | Actor instances admitted, active, and exited by Normal/Shutdown/Panic/Kill; handler starts, active handlers, completions, panics and interrupted futures. | | Delivery accounting | Enqueue acceptance/rejection, discarded queued messages, replies sent to channels, and reply receivers already dropped. Enqueue success does not promise handling or durable delivery. | +| Deferred replies (`spawned_deferred_*`) | `deferred_tokens_total` (successful `defer()`), `deferred_pending` (open slots with a live token, all actors), how each deferred slot ended: `replied`, `expired` (its deadline), `dropped`, `stopped` (owner exit or death), `abandoned` (caller gave up), each `_total`; `deferred_overloaded_total` (every `Overloaded` answer, at `defer()` or `ReplyTo::overloaded`); `deferred_reply_latency_seconds`, a sampled histogram from `defer()` to a delivered reply. | +| Jobs (`spawned_jobs_*`) | `jobs_started_total` (admitted), `jobs_running` (executing, cancelled or timed-out work included), terminal notifications `jobs_completed_total`, `jobs_panicked_total`, `jobs_deadline_exceeded_total`; `jobs_cancelled_total` (cancelled while still running: handle, key replacement, owner exit, caller give-up); `jobs_fenced_total` (job messages dropped at dispatch); `jobs_rejected_total` (at the credit bound or the native budget); `jobs_emits_total`; `jobs_unfinished_at_stop_total` (still running at the owner's cleanup deadline). | Counters end in `_total`. Duration histograms use fixed nanosecond buckets internally and seconds in Prometheus export; their count/sum/max describe sampled @@ -1162,6 +1554,11 @@ conservation relationships apply after operations quiesce: - handled = completed handlers + panicked handlers + aborted handlers + active; - observed requests = replied + timed out + actor stopped + cancelled + active; - spawned actor instances = exited + active. +- deferred tokens = replied + expired + dropped + stopped + abandoned + + pending (an `Overloaded` defer takes no token); +- started jobs = running + jobs whose execution ended (a job's terminal + notification is completed, panicked, deadline exceeded, or none when it was + cancelled first). An actor is active from creation of its lifecycle guard until terminal cleanup, including waiting for a blocking-pool slot, `started()` and `stopped()`. A runtime @@ -1182,11 +1579,17 @@ waits retain their existing first-poll timeout behavior. Synchronous protocol requests are observed by their blocking request; `Response::ready` does not count that result a second time. Manually constructed ready results are not requests. -`request_raw` returns the existing mode-specific raw receiver. It cannot observe -custom timeouts or how the caller waits without changing that API. Such requests +`send_request` returns a `ReplyReceiver`; it does not observe how the caller +waits on it, so it is not a caller-observed request. Its deadline clock is +part of the contract: awaited directly, a raw receiver is expired by the +owner's deadline heap on the real clock (the simulated one under +`spawned_sim`), not by a runtime timer, so it does not follow paused Tokio +time; that saves a timer per request on fan-out paths. Wrap it in +`Response::from(rx)`, or use `request`, when a test pauses Tokio time. Such requests still contribute message, handler, queue and reply-delivery metrics, but only a framework request/Response wrapper contributes caller-outcome/latency metrics. -Reply delivery means a successful channel send, not proof the caller consumed it. +Reply delivery means the reply was stored in the request's slot, not proof the +caller consumed it. Dropping/timing out a caller does not cancel accepted actor work. ### Collection and exporting limits @@ -1263,7 +1666,7 @@ out.expect_no_message(Duration::from_millis(20)).await; reply types still fail loudly. `allow_send::()` acknowledges one type only. - A message without a rule makes the mock panic with the type name and the - rule to add. The caller sees `ActorStopped`; `received()` lists it as + rule to add. The caller sees `ActorStopped(Panic(..))`; `received()` lists it as `Outcome::Unexpected` and `assert_no_unexpected()` fails the test. - `expect::(timeout)` returns the oldest not-yet-expected `M` (other types stay queued); `expect_no_message(window)`, `count::()`, `take_all::()`, @@ -1316,10 +1719,12 @@ These helpers fail with a report instead of hanging the test: | `wait_for_generation(&child, after, deadline)` | a supervised child does not reach a new generation, printing its snapshot | | `shutdown_supervisor(&supervisor, deadline)` | the supervisor report is incomplete, listing its children | | `shutdown_system(&system, deadline)` / `TestSystem::shutdown` | the `ActorSystem` report is incomplete, listing unfinished groups, actors, exits and failures | +| `in_actor(&actor, \|state, ctx\| ..)` | the actor is gone or the closure panicked; otherwise runs the closure in one of the actor's turns (after earlier sends) and returns its result, so a test can reach `ctx` (start a job, defer a reply) or inspect state without an ad hoc message or a `cfg(test)` bypass. It runs only after `started()` returned: work in `started()` a test must skip belongs in the constructor or a message | `TestSystem::new()` owns an `ActorSystem` with a two-worker async group named `test`, and starts actors and mocks on it. The `threads` module has the -blocking versions of `settle`, `exit_within`, `join_within` and `wait_until`. +blocking versions of `settle`, `exit_within`, `join_within`, `wait_until` and +`in_actor`. There is no generic "actor is idle" check: an actor's mailbox can receive from any sender at any time, so `settle` fences the senders the test controls. Every async wait in `tasks` (`expect`, `expect_no_message`, `wait_until`, @@ -1333,7 +1738,7 @@ capture it when called, since `#[track_caller]` does not reach into an | Fault | How | |---|---| -| Stop any actor with a chosen reason | `faults::exit(&handle, reason)`: `Normal` calls `stop()` and `Shutdown` calls `shutdown()`; both stop the actor even if it traps exits (unlike Erlang's `exit(Pid, normal)`, which a non-trapping process ignores). `Panic(msg)` arrives from a crashed linked peer and can be trapped. `Kill` uses a direct `ChildHandle::stop_with_reason(Kill)`, bypassing the target's trapping flag in both modes; its linked peers can trap the resulting notice. Because the panic travels through a link, the actor exits with `Panic("panic in handler: {msg}")`, and the default panic hook prints `msg`. An already-exited target is left alone. Never blocks the caller, even on a `current_thread` runtime. | +| Send an exit signal with a chosen reason | `faults::exit(&handle, reason)` is `ChildHandle::exit(reason)` (see [Exit signals](#exit-signals)): `Normal` is ignored by a non-trapping target; `Kill` ends the target `Killed`, trapping or not; any other reason (`Panic(msg)` included) ends a non-trapping target with exactly that reason, without `stopped()`, and reaches a trapping one as an `Exit`. For an untrappable stop that runs `stopped()`, use `ChildHandle::stop`/`stop_with_reason`. An already-exited target is left alone. Never blocks the caller, even on a `current_thread` runtime. | | Crash a supervised child and get its replacement | `tasks::restart_child(&child, reason, deadline)`: the returned instance always replaces the one it targeted, even if the child was already restarting; `deadline` bounds the whole call | | Slow dependency / forced request timeout | `Mock::delay::(d)` with a caller timeout shorter than `d` (`with_request_timeout`, `#[protocol(timeout = ..)]`) | | Lost message | `Mock::discard::()` (fire-and-forget messages: a request returning `Response<()>` still gets `()`; to lose a request, `delay` it past the caller's timeout) | @@ -1406,6 +1811,35 @@ assert_linearizable(&Kv, &recorder.take()); --- +## Experimental in 0.8 + +These items are public and tested but **may change before 1.0** (each is +waiting on a planned design or on a first user). Everything else follows the +release's semver promise. + +- The job API (`pipe*`, `reserve_*_job`, `JobOptions`, `JobCtx`), pending the + compute pool (rank 11b) and helper scopes (rank 15), and the + `Actor::MAX_NATIVE_JOBS` default (`min(4, available_parallelism)`). +- `PendingReplies` and `Context::deferred_outstanding`. +- `Context::exit_pending` (no OTP counterpart). +- `ChildSpec::max_outstanding` / `max_native_jobs`. +- The registry functions (`register`, `whereis`, `register_owned`, ..), which + rank 12 moves from process scope to `ActorSystem` scope. +- Helpers: `spawn_helper`, `spawn_blocking_helper`, `ScopedTask`, + `FactoryContext` (rank 15). +- Placement: `Backend::{Blocking, Thread}`, `pinned_group`, the affinity + functions, `thread_group_with_helpers`, `async_group_on`. +- The `diagnostics` feature and the `*-experiments` features; the + `lifecycle-trace` and `failpoints` features are test-only. +- `spawned_test::linearizability`. +- `spawned-rt`'s re-exports of Tokio and std types (beyond `run`, + `block_on`, `ctrl_c`, `time::Instant` and the simulation entry points). + +`start_linked` always starts an unmanaged `Backend::Async` actor, even from +a managed parent: the child is outside the parent's execution group and +ordered shutdown (the unified start API, planned for 0.9, will inherit the +parent's placement). + ## Further reading - [Semantics contract](design/semantics.md): every guarantee in one place (ordering, delivery, request outcomes, exit propagation, restarts, shutdown, timers, panics), each linked to what checks it, and what is not guaranteed. diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index f753e32b3..653fa7656 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -5,6 +5,332 @@ every change; this guide covers what you need to edit when upgrading. ## Unreleased +### 0.7 to 0.8: API call-site table + +0.8 is a breaking release. Every renamed or reshaped call is listed here +(old call, new call); the sections below explain the semantics. Rows marked +*(0.8 pre-release)* changed an API that only existed on the 0.8 branch +(ported clients used it), not in 0.7. + +| Old call | New call | +|---|---| +| `ctx.pipe(work_future, done)` *(0.8 pre-release)* | `ctx.pipe(JobOptions::new(), \|_\| work_future, done)` | +| `ctx.pipe_with(opts, work_future, done)` *(0.8 pre-release)* | `ctx.pipe(opts, \|_\| work_future, done)` | +| `ctx.pipe_with_ctx(opts, \|job\| fut, done)` *(0.8 pre-release)* | `ctx.pipe(opts, \|job\| fut, done)` | +| `ctx.pipe_keyed(key, work_future, done)` *(0.8 pre-release)* | `ctx.pipe(JobOptions::new().key(key), \|_\| work_future, done)` | +| `ctx.pipe_blocking_with(opts, \|job: &JobCtx<_>\| .., done)` *(0.8 pre-release)* | `ctx.pipe_blocking(opts, \|job: JobCtx<_>\| .., done)` (the job context is passed by value) | +| `ctx.pipe_reply(token, work_future, done)` *(0.8 pre-release)* | `ctx.pipe_reply(JobOptions::new(), token, \|_\| work_future, done)` | +| `ctx.pipe_reply_with(opts, token, work_future, done)` *(0.8 pre-release)* | `ctx.pipe_reply(opts, token, \|_\| work_future, done)` | +| `ctx.pipe_reply_blocking_with(opts, token, work, done)` *(0.8 pre-release)* | `ctx.pipe_reply_blocking(opts, token, work, done)` | +| `ctx.reserve_job()?.pipe_with(opts, work, done)` *(0.8 pre-release)* | `ctx.reserve_job(opts)?.pipe(\|_\| work, done)` | +| `ctx.reserve_blocking_job()?.pipe_blocking_with(opts, work, done)` *(0.8 pre-release)* | `ctx.reserve_blocking_job(opts)?.pipe_blocking(work, done)` | +| `JobOptions { deadline: Some(at), key: Some(k), emit_credits: n, cancel_on_caller_gone: true }` *(0.8 pre-release)* | `JobOptions::new().deadline(at).key(k).emit_credits(NonZeroU16::new(n).unwrap()).cancel_on_caller_gone()` | +| `JobOptions::default().deadline_after(d)` *(0.8 pre-release)* | `JobOptions::new().deadline(d)` (a `Duration`, an `Instant` or a `Deadline`) | +| `JobOutcome::Ok(v)` / `JobOutcome::Err(e)` / `outcome.into_result()` *(0.8 pre-release)* | `Ok(v)` / `Err(e)` / `outcome` (`JobOutcome` is `Result`) | +| `PipeError { kind: PipeErrorKind::Overloaded }`, `e.kind` *(0.8 pre-release)* | `PipeError::Overloaded`, `e` | +| `PipeReplyError::Failed { token, kind }` *(0.8 pre-release)* | `PipeReplyError::Failed { token, error }` (`#[non_exhaustive]`) | +| `DEFAULT_EMIT_CREDITS`, `default_max_native_jobs()` *(0.8 pre-release)* | removed from the public API (the defaults, 16 and `min(4, cores)`, are documented) | +| `actor.request_raw(msg)` (0.7: no deadline) | `actor.send_request(msg, Deadline::Never)` (or a `Duration` / `Instant`) | +| `actor.request_raw_until(msg, deadline)` *(0.8 pre-release)* | `actor.send_request(msg, deadline)` | +| `Response::from_with_timeout(actor.request_raw(msg), t)` | `Response::from(actor.send_request(msg, t))` | +| a `Recipient` call `recipient.request_raw(msg)` | `recipient.send_request(msg, deadline)` (`Deadline`); implementing `Receiver` is no longer possible (sealed, see below) | +| `reply.deadline()` / `token.deadline()` / `rx.deadline()` returning `Option` *(0.8 pre-release)* | return `Deadline` (`Deadline::Never` for none); forward with `other.send_request(msg, token.deadline())` | +| `token.with_deadline(at)` *(0.8 pre-release)* | `token.shorten_deadline(at)` (a `Duration`, an `Instant` or a `Deadline`) | +| `reply.has_caller()` *(0.8 pre-release)* | removed: `reply.defer()` returns `Err(DeferError::NoCaller)` for a send | +| `PendingReply::detached()` *(0.8 pre-release)* | `spawned_test::detached_reply()` | +| `PendingReplies::{purge, retained, live}` *(0.8 pre-release)* | `len()` / `is_empty()` (live tokens); purging is automatic on `insert` | +| `InsertError>` *(0.8 pre-release)* | `InsertError` (`#[non_exhaustive]`): match on `refused.kind()` (`InsertErrorKind`, also non-exhaustive) or take `let (kind, token) = refused.into_parts()`, so a wildcard arm still answers the token (e.g. `token.overloaded()`) instead of dropping it (`ReplyDropped`) | +| `DEFAULT_MAX_OUTSTANDING` *(0.8 pre-release)* | removed from the public API (`Actor::MAX_OUTSTANDING` defaults to 1024) | +| `system.shutdown(deadline)`, `supervisor.shutdown(deadline)`, `report.wait(deadline)` | unchanged calls; the `Duration` parameter is named `grace` (a deadline is an instant in 0.8) | +| `Down { monitor_ref, reason }` (pattern) | `Down { monitor_ref, reason, .. }`; `Down` gains `actor: ActorId` (the monitored actor) and, like `Exit`, is `#[non_exhaustive]` | +| exhaustive `match` on `SupervisorError`, `SystemError`, `HelperError`, `RegistryError`, `SupervisorStrategy`, `ExecutionProfile`, `ChildState`, `Backend`, `spawned_test::Outcome` | add a wildcard arm (all `#[non_exhaustive]`) | +| struct literal or exhaustive pattern of `ChildSnapshot`, `SupervisorReport`, `ActorExit`, `metrics::Snapshot`, `HistogramSnapshot` | read the fields; patterns need `..` (all `#[non_exhaustive]`) | +| `SupervisorConfig { strategy: s, max_restarts: n, ..Default::default() }` | `SupervisorConfig::new().strategy(s).max_restarts(n)` (or assign fields of `SupervisorConfig::default()`) | +| `Backoff { initial: a, jitter: false, seed: Some(7), ..Default::default() }` | `Backoff::new().initial(a).jitter(false).seed(7)` | +| `metrics::Config { timing_sample_every: n }` | `metrics::Config::new().timing_sample_every(n)` | +| `#[handler]` on an `#[actor]` method | `#[request_handler]` (answers with the return value) or `#[send_handler]` | +| `#[send_handler]` method returning a value | `#[request_handler]` (a send handler must return `()`; it used to answer a `request` with the value) | +| `impl Receiver for MyType` | not possible: `Receiver` is sealed (use `ActorRef`/`Context`, or `spawned_test`'s probe) | +| `spawned_concurrency::reply::{PendingReply, ReplyTo, ..}` | `spawned_concurrency::{PendingReply, ReplyTo, ..}` (the `reply` module is hidden; jobs types are under `tasks::`/`threads::`) | +| `actor.__in_turn(f)`, `reply.__discharge_stopped()`, `reply::reply_pair` *(0.8 pre-release, hidden)* | `spawned_test::{tasks, threads}::in_actor`, `spawned_test::reply_pair()`, `spawned_test::detached_reply()` | +| `spawned_concurrency::lifecycle_trace` without the `lifecycle-trace` feature | only with the feature (test-only, no stability promise) | +| `spawned_rt::time::{to_ticks, from_ticks, now_ticks, MAX_TICKS}` | still public for spawned-concurrency, `#[doc(hidden)]`, no stability promise | +| `Context::from_ref(&actor_ref)` | `actor_ref.context()` | +| `child_handle.shutdown()` | `child_handle.stop_with_reason(ExitReason::Shutdown)` | +| tasks: `actor.query().unwrap()` without `.await` (panicked) | off a runtime it now waits for the reply; inside one it still panics: `actor.query().await.unwrap()` | +| tasks: `response.is_ok()` / `is_err()` on an un-awaited response (answered `false`) | panics: `.await` it, or `response.into_result().is_ok()` off a runtime | + +### Exit signals, startup and runtime drops as in OTP (ranks 5e, 5f, 5g, targets 0.8, in progress) + +Design: [otp-exit-signals.md](design/otp-exit-signals.md) (its migration +table is the full list). What to edit: + +- **`match` on `ExitReason`**: add a wildcard arm (`#[non_exhaustive]`, new + `Killed` and `NoProc`). +- **`demonitor` flushes** (OTP `demonitor(Ref, [flush])`): a `Down` of that + monitor already in the mailbox is no longer handled. Code that demonitored + and then expected a racing `Down` (or tolerated one) needs no change; code + that relied on a queued `Down` arriving after `demonitor` must stop + demonitoring, or read `ChildHandle::exit_reason()`. +- **`max_restarts` with OneForAll/RestForOne counts rounds** (rank 5f): one + recovery round takes one grant, however many children it rebuilds (v0.7 + charged each rebuilt child). A cohort supervisor now tolerates more + failures for the same `max_restarts`; if you had raised it to cover + collateral rebuilds (for example `failures ร— children`), lower it to the + number of failures you mean to allow. OneForOne is unchanged. +- **Kill from outside**: `ChildHandle::kill()` is the untrappable kill; the + actor ends `Killed` and `stopped()` is skipped unless already running. + `stop_with_reason(Kill)` stays an administrative stop ending `Kill`. +- **`stopped()` no longer always runs.** It is skipped when the exit is begun + by a kill, an untrapped exit signal or link notice (a linked peer's + abnormal death), or a runtime drop (decision 2, as OTP skips `terminate`). + Cleanup that must run on every exit belongs in `Drop`. **To notice a + skip**, enable `debug` logs for `spawned_concurrency`: every skipped + `stopped()` logs `event = "stopped_skipped"` with the actor id and type, + `cause` (`kill`, `exit_signal`, `runtime_drop`) and `reason`. It is logged + for every skip, also for actors that do not override `stopped()`. The + remedy for a supervised actor is `ctx.trap_exit(true)` in `started()` + (below). + +> **Supervised and system-owned actors: `stopped()` is skipped on shutdown +> unless the actor traps exits.** Supervisor and system shutdown send the +> parent's `Shutdown` exit signal (OTP `exit(Child, shutdown)`). A +> non-trapping child ends `Shutdown` at once (after its current handler) +> **without `stopped()`**, as a gen_server that does not trap exits skips +> `terminate`. An actor that needs `stopped()` for cleanup on shutdown must +> trap exits, as in OTP; it then finishes the messages queued before the +> shutdown and runs `stopped()`. A child that has not finished within its +> `shutdown_timeout` is killed (`Killed`, `stopped()` skipped if it has not +> begun); cleanup counts as stuck only after a second `shutdown_timeout`. +> Unmanaged `.start()` actors have no parent and are unaffected. +> +> ```rust +> #[actor(protocol = StoreProtocol)] +> impl Store { +> #[started] +> async fn started(&mut self, ctx: &Context) { +> // v0.8: needed for stopped() to run on supervisor/system shutdown. +> ctx.trap_exit(true); +> } +> +> #[stopped] +> async fn stopped(&mut self, _ctx: &Context) { +> self.flush_to_disk().await; +> } +> } +> ``` +> +> A trapping actor also receives linked peers' deaths as `Exit` messages +> (`exit_received`) instead of dying with them; trap only where that is +> intended, or move the cleanup into `Drop`. +> +> A trapping actor still handles the messages queued before the shutdown. To +> refuse that late work, check the new `ctx.exit_pending()` (both modes): it +> is `true` once the actor's exit has begun or the parent's exit is queued +> behind the current message (a peer's trapped `Exit` does not count). It is +> advisory: a signal can arrive right after it returns `false`. + +- **Runtime drop** reports `Killed` (was `Kill` in tasks mode, + `Panic("unexpected framework panic")` in threads mode). +- **Late link or monitor**: to an actor past its death point, `monitor` + delivers `Down { reason: NoProc }` and `link` delivers `NoProc` (a + non-trapping caller exits `NoProc`). Use `ctx.try_link` to get + `Err(NoProc)` instead, and `ChildHandle::exit_reason()` for the real + reason. `link` from an actor whose exit already began is inert. +- **A late signal during exit**: a non-`Normal` link notice or exit signal + that reaches a non-trapping actor while it exits (for example during its + `stopped()`) now replaces its reason. The regression test + `late_link_failure_cannot_relabel_an_already_requested_normal_stop` was + inverted deliberately into + `late_link_failure_relabels_an_already_requested_normal_stop`. +- **Panic in `stopped()`** after an abnormal reason now becomes the reason + (`Panic("panic in stopped(): ..")`), except after a kill. +- **Destructor panics** (actor state, queued messages) after the death point + no longer change the exit reason: a `Normal` exit stays `Normal` and + `ChildHandle::cleanup_failure()` reports the failure + (`mailbox_cleanup_failure()` is a deprecated alias). A Transient child + restarts on an abnormal reason **or** a cleanup failure, also after + `Shutdown`; `ChildSnapshot::last_error` holds the failure. +- **`start_linked` children that trap exits** end when their parent ends, + with the parent's reason (`Normal` included), through `stopped()`; + `exit_received` no longer sees the parent's exit. Use `ctx.link` instead + of `start_linked` to observe the parent's exit as an `Exit`. +- **`spawned-test` `faults::exit`** now sends the exact exit signal + (`ChildHandle::exit`): `Normal` is ignored by a non-trapping target, + `Panic(msg)` ends it with exactly `Panic(msg)`, `Kill` ends it `Killed`, a + trapping target receives an `Exit`. +- **Supervisor start (rank 5f):** any first-start failure of a child (factory + error or panic, panic or stop in `started()`, `startup_timeout`, placement) + fails `SupervisorBuilder::start()`, whatever its restart type, with no + opt-in retry (decision 7), as `SupervisorError::FailedToStartChild { child, + cause, rollback_failures }`; match on it (or keep treating any `Err` as a + failed start). Replacements still retry within the restart budget. A ready + child that exits while later siblings start is no longer restarted at once + (OneForOne) or fatal (OneForAll/RestForOne): its exit is processed after + the initial start completes. `start()` now returns its error after the + rollback, so it can take up to two `shutdown_timeout`s per started child; a + child whose cleanup is stuck is reported `Failed` and its work stays owned + by the system. **ethrex**: + the Transient discovery child's factory (`DiscoveryServer::build`) can fail + at boot when pinging a bootnode fails; v0.7 retried it, v0.8 fails + `start()` and `init_network` panics on `.expect("Network starts")`. Fix in + ethrex: make the bootnode ping best-effort (log and continue) so the + factory does not fail on it, or retry `start()` at application level. +- **Caller errors carry the reason (with rank 11, decision 4).** + `ActorError::ActorStopped` is now `ActorStopped(ExitReason)`: the actor's + final exit reason, the one `wait_exit`, `Down` and linked actors see + (`Killed` after `kill()`, the signal's reason after an untrapped exit + signal, `Panic(..)` after a crash, `Normal`/`Shutdown` after a stop). A + `send` or `request` to an actor past its death point returns the new + `ActorError::NoProc` instead; one sent while the actor is still exiting + (for example during its `stopped()`) is admitted and answered at the death + point. Patterns change from `Err(ActorError::ActorStopped)` to + `Err(ActorError::ActorStopped(_))` (or a specific reason); code that + treated `ActorStopped` as "the actor is gone" for sends should also handle + `NoProc`. A request whose handler panicked, or a deferred token dropped + while the actor exits, is no longer answered at once: it is answered at the + death point (after `stopped()`) with the final reason, or by the caller's + own deadline first. With `metrics`, rejected requests count in the new + `requests_noproc_total`. `ActorError` stays `#[non_exhaustive]`; instead + of an exhaustive match, classify with `target_gone()` (`ActorStopped(_)`, + `NoProc`), `is_timeout()` (`RequestTimeout`), `is_transient()` + (`RequestTimeout`, `Overloaded`: retrying the same target may succeed; + not `ReplyDropped`) and `exit_reason()` (the reason of `ActorStopped`). + A variant added later is classified by these helpers. **ethrex**: + `peer_handler.rs:1251` matches `ActorError` exhaustively on purpose, so it + no longer compiles: add arms for `ActorStopped(_)` and `NoProc`, or replace + the match with the helpers above; two tests that construct + `ActorError::ActorStopped` must pass a reason (for example + `ActorStopped(ExitReason::Normal)`). **ethlambda**: one test mock + constructs `ActorStopped`; pass a reason the same way. A `spawned-test` + `Probe` closed with `close()` now returns `NoProc`, and a probe without a + reply answers `ActorStopped(NoProc)`. + +### Deferred replies (rank 11, targets 0.8, in progress) + +This is a breaking release. The notes below grow with the implementation: + +- Request timeouts follow OTP's `gen_server:call`: only the caller's expiry + (its timer, the owner's deadline heap for a raw `send_request` deadline, + or a deadline check such as `is_closed()`) times a request out; a reply + reads no clock. Code that relied on a reply at or past the deadline being + refused (`Delivery::Expired`) must not: a reply that lands after the + deadline but before the caller's expiry runs is `Delivered` and the caller + gets the value. `Expired` still means the caller already saw + `RequestTimeout`. + +- `ActorError` is `#[non_exhaustive]` and gains `ReplyDropped` and + `Overloaded`; add a wildcard arm to exhaustive matches. + +**Handlers.** `#[actor]` code is unchanged. A manual `impl Handler` now +takes the request's reply obligation and returns `()`: + +```rust +// before +impl Handler for Counter { + async fn handle(&mut self, _: GetCount, _: &Context) -> u64 { self.count } +} +// after +impl Handler for Counter { + async fn handle(&mut self, _: GetCount, reply: PendingReply<'_, u64>, _: &Context) { + reply.reply(self.count); + } +} +``` + +A handler that returns without answering a request now gives the caller +`ReplyDropped` (a panic still gives `ActorStopped`, with the final reason at +the actor's death point). Sends reach the same +`handle` with a detached `PendingReply`. To call a manual handler directly in +a test, use `spawned_test::reply_pair()`. + +**Timeouts.** The request timeout starts at send, not at first poll. + +**Raw API.** `request_raw(msg)` is now `send_request(msg, deadline)` +(OTP's `gen_server:send_request/2`, plus an explicit deadline; OTP's +`/4` form takes a label and a request-id collection instead). It takes the deadline explicitly: a +`Duration` from now, an `Instant`, or `Deadline::Never`. **0.7's +`request_raw` had no deadline**: `send_request(msg, Deadline::Never)` keeps +that unbounded wait, and `send_request(msg, DEFAULT_REQUEST_TIMEOUT)` gives +the 5 s `request` uses. It returns a `ReplyReceiver` (a future in tasks +mode, `recv()` in threads mode) yielding `Result`. +`Response::from_with_timeout` is removed: `Response::from(actor.send_request(msg, t))`. +`Receiver` is sealed: call `send_request(msg, Deadline)` through a +`Recipient`; a type of your own cannot implement it (wrap an `ActorRef` +or use `spawned_test`'s probe instead). A `recv_timeout` on the old native +receiver has no counterpart: the deadline is fixed when the request is +sent. + +**Metrics.** `metrics::Snapshot` gains `requests_reply_dropped_total` and +`requests_overloaded_total`, the `deferred_*` and `jobs_*` fields (see the +GUIDE's Metrics section) and the `deferred_reply_latency` histogram. It is +`#[non_exhaustive]` and read-only: take one from `metrics::snapshot()` and +read its fields; a struct literal or an exhaustive pattern no longer +compiles (patterns need `..`). + +**Jobs (API refinements).** + +- One spawn per shape: `pipe` (async), `pipe_blocking`, `pipe_reply`, + `pipe_reply_blocking`, each taking `JobOptions` first (as OTP's `*_opt` + calls take an option list). The work of every form receives its + `JobCtx` (by value; `|_| fut` when unused). See the table above. +- `JobOptions` is `#[non_exhaustive]`: build it with `JobOptions::new()` + and the `key`, `deadline`, `emit_credits` and `cancel_on_caller_gone` + methods. `deadline` takes a `Duration`, an `Instant` or a `Deadline`; + `emit_credits` takes a `NonZeroU16`. `cancel_on_caller_gone` applies to + `pipe_reply*` only (a `debug_assert!` in the plain forms). +- `JobOutcome` is `Result`: `?` and the `Result` + combinators work, `into_result()` is gone. `JobError` and `PipeError` + (now the enum itself, without the `kind` wrapper) are + `#[non_exhaustive]`: add a wildcard arm. +- `tasks` mode: `pipe_blocking` / `pipe_reply_blocking` run on + Tokio's blocking pool. Remove a `pipe(.., |_| async { spawn_blocking(f) + .await })` workaround (and its `resume_unwind` of the join error): + `pipe_blocking(opts, move |_job| f(), done)` reports a panic as + `JobError::Panicked`. **Size the native budget:** the default is + `min(4, available_parallelism)` blocking jobs per actor, easy to outgrow + (one client port needed 352), and under load its only symptom is + `PipeError::NativeBudget` refusals (the first one per actor is logged + as `native_budget_exhausted`). Raise it with + `#[actor(max_native_jobs = N)]` (or `ChildSpec::max_native_jobs(N)` for a + supervised child) instead of avoiding native jobs. A Tokio runtime being + dropped now waits for blocking jobs still running, as for any + `spawn_blocking`. +- To keep "exactly one result" when a job is refused, reserve first instead + of cloning ahead: `let Ok(permit) = ctx.reserve_blocking_job(opts) else { + /* report with the values you still own */ }; + permit.pipe_blocking(work, done);` (`reserve_job` for async + jobs). Spawning from a permit cannot fail. +- A job that re-scheduled its messages with `send_after` (unfenced) can use + `job.emit_at(at, msg)` / `emit_at_blocking`: fenced with the job, counted + against its emit credits, and the terminal message waits for it. +- Use `spawned_concurrency::Instant` instead of `spawned_rt::time::Instant`. + Job deadlines are hard (later emits fail, the result is dropped); for a + soft deadline, check the time in the work (see the GUIDE). + +**Send-only manual handlers.** A manual `impl Handler` whose `Result` is +`()` no longer needs the `_reply: PendingReply<'_, ()>` parameter: put +`#[send_handler]` (from `spawned_concurrency`) on the impl and write +`handle(&mut self, msg, ctx)`. Without it, a handler that drops its `()` +reply gives a `request` caller `ReplyDropped`; with it, the caller gets +`Ok(())`. + +**Tests.** Replace test-only "run this closure in the actor" messages and +`cfg(test)` bypasses of handlers with `spawned_test::tasks::in_actor(&actor, +|state, ctx| ..)` (`threads::in_actor` in threads mode). `in_actor` runs in +a turn, so only after `started()` returned: it cannot replace a bypass +inside `started()` itself. If `started()` does work a test must skip (for +example `if handshake().await.is_err() { ctx.stop() }`), keep that work +injectable instead: pass the handshake (or its outcome) in the actor's +constructor, or move it to a message the test does not send, so the test +builds the post-handshake state directly. + +**Lifecycle.** At death the registry closes (open deferred tokens answer +`ActorStopped`), then the mailbox is disposed, then actor state is dropped: +a state destructor that sends to its own actor now fails (with `NoProc` +once the exit-signals change above is in) instead of enqueueing into a +mailbox about to be dropped. + **Simulation builds only (`--cfg spawned_sim`; determinism D1-D3).** - `sim_test` / `SimTest::run`, `sim::run_once`, `run_with`, `shrink` and @@ -486,9 +812,9 @@ The `#[actor]` macro supports three handler attributes: |-----------|---------| | `#[request_handler]` | Messages that expect a reply (`Response`) | | `#[send_handler]` | Fire-and-forget messages (no return or `-> ()`) | -| `#[handler]` | Generic โ€” works for either kind | +| `#[handler]` | Generic โ€” works for either kind (removed in 0.8) | -All three generate the same `impl Handler` code. The distinction is semantic: use `#[request_handler]` and `#[send_handler]` to document intent, or `#[handler]` when you don't want to commit to a specific kind. +In 0.5 to 0.7 all three generate the same `impl Handler` code. Since 0.8, `#[handler]` is removed and a `#[send_handler]` method must return `()` (see the 0.7 to 0.8 table). Handler signature: `fn name(&mut self, msg: MessageType, ctx: &Context) -> ReturnType` diff --git a/docs/TESTING.md b/docs/TESTING.md index 1d7fd39ca..be48ecb1a 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -180,7 +180,7 @@ Status of each layer. "Planned" items are tracked in the | Design | TLA+ and TLC | Holes in lifecycle, restart and shutdown rules | `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` models in `verification/tla`, run in CI | | Design, unbounded depth | Apalache inductive invariants | An invariant that a state TLC never reached would break | Safety invariants of all four models proven inductive at fixed sizes, with negative controls; `tla-apalache` CI job | | Design, any size | TLAPS or Veil | An invariant that fails only for larger configurations | Planned ([verification tools](research/verification-tools.md)) | -| Spec and code | Trace validation | The implementation doing something the model forbids | Real OneForOne, cohort and nested traces plus invalid-trace and model-mutation controls | +| Spec and code | Trace validation | The implementation doing something the model forbids | Real OneForOne, cohort, nested, deferred-reply, job and base-lifecycle (exit signals, links, monitors, D, publication) traces plus invalid-trace and model-mutation controls | | Spec and code coverage | Trace coverage (`verification/tla/trace_coverage.py`) | Model actions, states and transitions no test ever reaches | Report on every PR (actions covered by the dedicated trace tests may not regress); whole test suite weekly. See [trace coverage](#trace-coverage) | | Spec and code in sync | `scripts/check-model-sync.py`, `verification/model-map.toml` | A PR changing modelled code without its model; a trace event no trace spec replays; a model no real run is checked against; trace coverage going down | `model-sync` job on every PR. See [model sync](#model-sync) | | Primitives | Loom | Atomic and memory-ordering bugs in cancellation, ready gate and wakeups | In CI; expand to reply slots and mailbox publication versus shutdown | @@ -224,7 +224,7 @@ The fuller rationale is archived in the ```sh python3 scripts/check-api.py features -python3 scripts/check-api.py semver --baseline v0.6.0 +python3 scripts/check-api.py semver # baseline and release type from verification/api/release.json ``` Install cargo-hack 0.6.46 for the first command. CI pins cargo-semver-checks @@ -282,7 +282,7 @@ This generated suite uses zero backoff and a high, unexhausted restart budget; it does not claim budget-exhaustion or shutdown-during-backoff coverage. Those are controlled cases in `supervision` (`restart_budget_is_shared_between_siblings_and_exhaustion_closes_tree`, `shutdown_during_backoff_prevents_new_factory_invocation`) and -`supervision_strategies` (`cohort_reconstruction_consumes_shared_factory_attempt_budget`). +`supervision_strategies` (`cohort_reconstruction_charges_one_restart_per_round`). ```sh cargo test --locked -p spawned-concurrency --test supervisor_policies @@ -451,10 +451,11 @@ handler panic. The fixed regression case exercises every termination mode. - A request fence observes every earlier message from the same sender in order, exactly once. Dropping a reply receiver does not cancel queued work. -- A request racing shutdown either returns its matching reply or ActorStopped. +- A request racing shutdown returns its matching reply, `ActorStopped` or + `NoProc`. Accepted sends alone do not guarantee execution across termination. - Termination resolves within a five-second test watchdog. Post-exit requests - return ActorStopped. + return `NoProc`. - Each uncancelled monitor observes the matching exit reason, including when registration races termination. Unknown or duplicate observed notifications fail the test. Clean shutdown and injected handler panic remain distinct. @@ -707,7 +708,8 @@ simulated at all): tests that need OS threads or a real Tokio runtime, in `cancellation_safety`, `cross_backend` (non-async placements and the fan-out and monitor workloads), `dedicated_runtime`, `delivery_simulation`, `execution_groups`, `execution_profiles`, `failure_windows`, `leaks`, -`mailbox_drop_panic`, `nested_supervision`, `panic_safety`, +`mailbox_drop_panic`, `nested_supervision`, `otp_exit_signals` (whole file; +`exit_signals_sim` explores its orderings), `panic_safety`, `process_failures`, `protocol_compile`, `request_histories`, `resource_cleanup`, `semantics_gaps` (its `tasks` and `tasks_contract` modules), `shutdown_publication`, `stack_budget`, `startup_admission`, `supervision_strategies` and `supervision`. `leaks` replaces its excluded @@ -723,6 +725,45 @@ RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ cargo test -p spawned-concurrency --test supervision --test sim ``` +`deferred_sim` explores deferred-reply and job races with PCT (`d=3`) and +bounded DFS (`bound=3`), 300 runs each: a stored token's reply against its +deadline, the owner's stop and the caller dropping its receiver (the caller +sees `Ok` exactly when the reply reports `Delivered`); keyed replacement +against the old job's completion (no result of a replaced job after its +replacement starts); emits against the terminal message and a cancel. Planted +bugs (no cancel fence at dispatch; a late reply reported `Delivered`) are +found within the first runs. + +```sh +RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ + cargo test -p spawned-concurrency --test deferred_sim --test deferred_replies --test deferred_jobs +``` + +`exit_signals_sim` explores the exit-signal orderings with PCT (`d=3`, 1,000 +runs) and bounded DFS (`bound=3`, `depth=300`, up to 300 runs for each of +input seeds 0-11): a kill against an administrative stop's slow `stopped()` +(skipped before it begins, finishes during it, ignored after the death point; +a monitor sees the same reason), a late untrapped signal against the death +point (replaces the reason before it; link and monitor agree), a `link` + +`monitor` or `try_link` against the target's death (one notice each, the real +reason or `NoProc`, the same for both registrations of one turn), `unlink` + +`demonitor` against delivery (nothing after a marker sent right after the +fence), a supervisor's shutdown against a trapping child's queued messages +(handled in order, `exit_pending` monotone, `stopped()` runs, no +`exit_received`), and a ready child's kill against the tree's initial commit +(start succeeds, one restart, after the slow sibling is ready). Each test +counts the outcomes its assertions distinguish and every one is reached. The +DFS searches are exhausted for every seed of the four single-actor workloads +and for 10 of 12 seeds of the supervisor shutdown one; the startup workload +reaches its 300-run budget for every seed. Planted +bugs (a late signal ignored, a kill that does not skip `stopped()`, a monitor +that ignores the death point) are found in the first run. + +```sh +RUSTFLAGS="--cfg spawned_sim --cfg tokio_unstable" CARGO_TARGET_DIR=target/sim \ + cargo test -p spawned-concurrency --test exit_signals_sim -- --nocapture +``` + ### Simulated suites in CI The `sim` job in `.github/workflows/ci.yml` builds with the cfg and runs @@ -968,8 +1009,8 @@ input and, for random schedules, its tape: This is generated, model-checked scenario testing on real runtimes, not an exhaustive scheduler; the coverage-guided fuzzer above runs on the simulator only. It does not cover bounded mailboxes or supervisor restart policies. -Demonitor races permit an already-queued notification; tests use bounded -observation windows and do not claim exhaustive delivery verification. +Demonitor flushes a queued `Down` (X2); these campaigns still observe it +through bounded windows and do not claim exhaustive delivery verification. Do not interpret a passing campaign as proof of race freedom. ## Additional checks @@ -996,6 +1037,44 @@ The custom cfg isolates instrumentation from Tokio's own Loom configuration. These models cover registration/cancellation races and reentrant callbacks, not the entire actor runtime. Do not run ordinary actor tests with this cfg. +The reply and job cores of deferred replies share their source with Loom +models that inject Loom's synchronization types (`super::sync`): + +```sh +RUSTFLAGS="--cfg spawned_scheduler_loom" cargo test --release -p spawned-concurrency \ + --no-default-features --test reply_slot_loom --test registry_turn_loom --test job_core_loom \ + --test link_fence_loom +``` + +`reply_slot_loom` covers the packed slot word (reply, expiry, shortening, +abandon, close, defer at the bound, linked jobs, parked receivers); +`registry_turn_loom` the in-turn drop commit against turn end, exit and death; +`job_core_loom` job admission and rollback at the credit and native bounds, +keyed replacement of a completed job, emits against the terminal notification +and cancel, a delayed emit (`emit_at`) firing against the work's finish (the +terminal held until it fires) and a cancel, native completion against the +owner's cleanup join, and a job's +final destruction after a `Weak` upgrade (modelled `Arc`/`Weak`; the +destruction takes the job set's locks, so running it under one is a Loom +deadlock). Each model was checked against a planted bug. + +`link_fence_loom` shares the link table (`src/link/table.rs`: edge +generations, the two-lock install, receiver-side fenced delivery, the close at +the death point) and the observer's monitor table (`src/monitor/fence.rs`) +with production in the same way; `ActorId`, `ExitReason` and the receiver are +stand-ins, and applying a notice is an append under the receiver's lock. Its +nine tests are explored without a preemption bound (5 to 382 executions +each): at most one notice per edge generation, none after the receiver's +`unlink` returned or after its own death point, exactly one notice (real +reason or `NoProc`) for a `link` racing the peer's death and none for a +`try_link` after it, a stale generation ignored after unlink and relink, one +edge from concurrent links; at most one `Down`, none after `demonitor` or the +observer's close returned, exactly one (real reason or `NoProc`) for a +`monitor` racing the target's death. Seven planted bugs (no generation check, +unlink keeping the receiver's record, close keeping records, install ignoring +the peer's death, `Down` enqueued outside the lock, registration ignoring the +death point, a close that keeps the table open) each fail at least one test. + Run sustained actor creation, traffic, monitoring, and shutdown: ```sh @@ -1262,6 +1341,10 @@ seeded random runs, and TLC checks every recorded trace is a behavior of the model. Hand-edited traces are rejected at the edited event. Trace validation found one model gap (a running child may exit with `Shutdown` while the supervisor is open, via `ChildHandle::shutdown`); no implementation deviation. +`lifecycle_trace` does the same for the base actor lifecycle against +`Lifecycle.tla` (exit signals, stops, links, monitors, requests, `stopped()` +run or skip, the death point, runtime drop and publication, in both modes; +eleven edited traces must be rejected). Apalache checks that the safety invariants of `Supervision`, `Cohorts`, `NestedSupervision` and `Coordinator` are inductive (`Init => IndInv`, @@ -1273,7 +1356,7 @@ controls must break induction. See [the model README](../verification/tla/README ```bash nix run .#check-tla # model check (TLC via the flake) nix run .#check-apalache -- --negative-controls # inductive invariants (Apalache) -cargo test -p spawned-concurrency --features lifecycle-trace --test supervision_trace +cargo test -p spawned-concurrency --features lifecycle-trace --test supervision_trace --test lifecycle_trace nix run .#check-tla -- --traces target/lifecycle-traces ``` @@ -1622,8 +1705,15 @@ and prints the seed when a test fails. Configuration is process-global. | `system.reserve` | yes | Placement failure (`SystemError::Capacity`) | | `registry.before_register` | yes | Shutdown racing managed name registration | | `registry.before_unregister` | no | Slow removal of an owned name before a replacement starts | +| `actor.at_death` | no | An external reply racing the death point after `NoProc` is observable (tasks and threads) | +| `actor.exit_before_deliver` | no | A cloned context's exit signal racing its sender's death (tasks and threads) | | `link.before_install` | no | Peer exiting while a link is installed (tasks and threads) | | `monitor.before_install` | no | Target exiting while a monitor is installed (tasks and threads) | +| `job.emit_due` | no | A scheduled `emit_at` coming due during the owner's exit: its payload is held until every caller is answered (`emit_due_during_exit_*`) | +| `job.native_worker_unused` | no | Threads mode: the parked thread of a dropped, unused blocking-job permit held at its exit; the native slot is already back (`an_unused_permit_returns_its_slot_without_its_thread`) | +| `reply.close_upgraded` | no | The death point's close holding an already-answered slot while its caller drops the receiver: the unread reply is destroyed only after every caller is answered (tasks and threads) | +| `reply.recv_parked` | no | Handshake: a blocking `recv()` is parked, so only a notification wakes it (discharge notifies parked receivers before any waker; tasks and threads) | +| `monitor.down_before_dispatch` | no | `demonitor` on another thread returning while the monitor's `Down` is in dispatch: it is flushed (tasks and threads) | `tests/failpoints.rs` drives every site and checks invariants through public APIs: restarts follow the budget (generation count equals injected failures @@ -1848,8 +1938,13 @@ time on the simulator, and only reached by a bug on Tokio, since every such wait is a handshake, never a fixed sleep. Tests on paused Tokio time (`protocol_timeouts`, `semantics_gaps`) are left alone: paused time auto-advances while work runs outside Tokio, which could fire the bound early. -Unit tests inside `concurrency/src` do not use it yet (see "Planted-bug -sensitivity" for the ones that still hang under a planted bug). +Unit tests inside `concurrency/src` that start, call, join or wait on actors +are bounded the same way (`concurrency/src/test_wait.rs`, 120 s): +synchronous tests run their whole body, Tokio runtime and its teardown +included, through `test_wait::bounded`, on a thread named after the test, and +fail with that name if it does not finish; `#[tokio::test]` bodies go +through `test_wait::within`. A panic in the body (an assertion, or a +`#[should_panic]` test's expected panic) is re-raised unchanged. ### Single-CPU runs diff --git a/docs/design/deferred-replies-client-mapping.md b/docs/design/deferred-replies-client-mapping.md index e9f4c413c..3fe85a7de 100644 --- a/docs/design/deferred-replies-client-mapping.md +++ b/docs/design/deferred-replies-client-mapping.md @@ -86,8 +86,10 @@ async fn request(&mut self, msg: Request, reply: PendingReply<'_, Message>, ctx: let Ok(token) = reply.defer() else { return }; // Overloaded (caller told) or Closed (caller gone) match state.current_requests.insert(id, token) { Ok(()) => state.outbox.push(msg.message), // see note on the socket send - Err(InsertError::Duplicate(token)) => drop(token), // the new caller is told; the old one kept - Err(InsertError::Full(token)) => token.overloaded(), // caller gets Err(Overloaded) + Err(refused) => match refused.into_parts() { + (InsertErrorKind::Duplicate, token) => drop(token), // the new caller is told; the old one kept + (_, token) => token.overloaded(), // Full or a later reason: caller gets Err(Overloaded) + }, } } // response path @@ -203,11 +205,11 @@ Gaps: ### After (v1, before the rank 11b pool) ```rust -let opts = JobOptions { deadline: Some(now + VERDICT_DEADLINE), ..Default::default() }; -match ctx.pipe_blocking_with(opts, move || object.stateful_checks(&store).map(|o| (o, object)), - move |res| GossipVerdict { id, res }) { +let opts = JobOptions::new().deadline(VERDICT_DEADLINE); +match ctx.pipe_blocking(opts, move |_| object.stateful_checks(&store).map(|o| (o, object)), + move |res| GossipVerdict { id, res }) { Ok(_job) => {} - Err(PipeError { kind: PipeErrorKind::Overloaded | PipeErrorKind::NativeBudget }) => { + Err(PipeError::Overloaded | PipeError::NativeBudget) => { report(server, id, Ignore(Overloaded)); forward(..); } Err(e) => { report(server, id, Ignore(Internal)); warn!(?e); } @@ -215,9 +217,9 @@ match ctx.pipe_blocking_with(opts, move || object.stateful_checks(&store).map(|o impl Handler for P2PServer { async fn handle(&mut self, msg: GossipVerdict, _: PendingReply<'_, ()>, _: &Context) { let outcome = match msg.res { - JobOutcome::Ok((outcome, _object)) => outcome, - JobOutcome::Err(JobError::Panicked(_)) => Ignore(Internal), - JobOutcome::Err(JobError::DeadlineExceeded) => Ignore(Timeout), // new: hung checks still yield + Ok((outcome, _object)) => outcome, + Err(JobError::DeadlineExceeded) => Ignore(Timeout), // new: hung checks still yield + Err(_) => Ignore(Internal), // a panic }; self.settle(msg.id, outcome, ..); } @@ -274,11 +276,10 @@ Hazards: ```rust fn start_aggregation_session(&mut self, slot: u64, ctx: &Context) -> Result<(), PipeError> { let snapshot = self.snapshot_for(slot); - let opts = JobOptions { key: Some(AGGREGATION), deadline: Some(interval_2_boundary(slot)), - ..Default::default() }; + let opts = JobOptions::new().key(AGGREGATION).deadline(interval_2_boundary(slot)); // Keyed: cancels the prior session's job and fences its queued messages; // no join in the turn (the job set joins at stop, with a deadline). - ctx.pipe_blocking_with(opts, move |job: &JobCtx| { + ctx.pipe_blocking(opts, move |job: JobCtx| { for proof in snapshot.jobs() { if job.is_cancelled() { break } job.emit_blocking(AggregateProduced { slot, proof: prove(proof) })?; // fenced per emit @@ -317,7 +318,7 @@ Also: `PendingReplies` (sections 1 and 2). Expired requests are still dispatched (OTP), so handlers check `reply.is_closed()` or rely on `defer()` returning `Closed`. -- `pipe_blocking_with`, `JobOptions { key, deadline }` and +- `pipe_blocking`, `JobOptions::new().key(..).deadline(..)` and `JobCtx::emit_blocking` (sections 3 and 4). - Rank 11b for pool-based admission classes (section 3). - Ethrex is on spawned 0.5.0. These rewrites assume it first migrates to v0.7 diff --git a/docs/design/deferred-replies.md b/docs/design/deferred-replies.md index 65b6a23de..213d9a5b2 100644 --- a/docs/design/deferred-replies.md +++ b/docs/design/deferred-replies.md @@ -1,6 +1,6 @@ # Deferred replies and pipe-to-self: design contract -**Status: design in progress, not implemented.** +**Status: implementation in review (PR #239); see [Implementation notes](#implementation-notes).** - Covers ROADMAP rank 11 and the rank 11a decisions. The rank 11b compute pool is deferred to 11b. @@ -32,8 +32,8 @@ See [Differences from OTP](#differences-from-otp). | `handle_call` returns `{noreply, State}` and keeps `From` | the handler receives a `PendingReply<'_, R>`; `defer()` gives a storable `ReplyTo` | | `gen_server:reply(From, R)` | `reply_to.reply(value)` | | `handle_cast` | the same handler, reached by `send`; the `PendingReply` is detached | -| delegating with `From` | `ctx.pipe_reply(token, other.request(msg), ..)`; the obligation stays here | -| `Task.Supervisor.async_nolink` + `handle_info` | `ctx.pipe(work, done)`; the result or a caught panic arrives as one message | +| delegating with `From` | `ctx.pipe_reply(opts, token, \|_\| other.request(msg), ..)`; the obligation stays here | +| `Task.Supervisor.async_nolink` + `handle_info` | `ctx.pipe(opts, work, done)`; the result or a caught panic arrives as one message | | `gen_server:call/3` timeout (default 5000, or `infinity`) | one deadline fixed at send (default 5 s, or `Deadline::Never`) | | a call times out; the server still handles it; the late reply is dropped | the same: expired requests are dispatched, and a late reply is a no-op | | call monitors the server and fails when it dies | the owner's death discharges every token it holds | @@ -82,16 +82,47 @@ compare-and-swap on that word, so "install a credit only if `OPEN`" and ```text word: AtomicU64 = state:4 | TOKEN:1 | REG:1 | JOBS:16 | (spare) - state: OPEN | WRITING | REPLIED | DROPPED | OVERLOADED | TIMED_OUT | STOPPED | ABANDONED + state: OPEN | REPLIED | DROPPED | OVERLOADED | TIMED_OUT | STOPPED | ABANDONED TOKEN: a deferred token holds this request's credit (set by defer()) REG: listed in the owner registry (set together with TOKEN) JOBS: running linked jobs (pipe_reply); MAX_OUTSTANDING is limited to 65535 at configuration, so the field cannot overflow deadline: AtomicU64 rt clock ticks; u64::MAX = never; only decreases -value: written only by the thread that moved the word to WRITING -waker: AtomicWaker (tasks) / Mutex+Condvar (threads) +inner: one mutex (the slot lock) over + value: written only by the reply that won the word, before the + CAS that publishes REPLIED + waker: the caller's task waker, or the unparker of a thread + blocked in recv() (then `parked` is set, so the death + point's discharge wakes it before any other waker) + side: boxed, created on demand: by defer() (the owner's credit + pool and the token's watchers: ReplyTo::closed, + caller-gone job cancellation) and by the owner's close + (its final reason, stored before the STOPPED CAS so the + caller reads it from the slot); a plain request never + allocates it +owner: none outside lifecycle-trace builds (whose events name it): a + request takes no shared reference count on its owner. A raw + request with a deadline (`send_request`) records the owner's + registry key in the word's top 40 bits; its receiver, awaited and + still pending, finds the owner by that key and puts the deadline + in the owner's heap once (a request answered first never touches + the heap, and no heap entry outlives a slot answered before it). + An undispatched envelope is answered through the owner's mailbox + disposal at the death point (`discharge(replies)`). PARKED (bit + 22) and the key are word bits no transition reads or changes; as + bits, they keep the slot inside the allocator's small size + classes + the slot lock also orders a shortening against a reply's + deadline read ``` +A terminal transition takes the waker under the slot lock after its CAS; +a receiver checks the word under that lock before it registers, so either +the transition finds its waker or the check sees the terminal state (no +lost wakeup; `tests/reply_slot_loom.rs`). A blocking receiver parks its +thread (no condvar). An `AtomicWaker` beside a value mutex was measured and +costs more per request than one lock for both. + **Credits held by a slot:** ```text @@ -114,29 +145,30 @@ operation first reads `now`, and if `now >= deadline` it performs | `defer()` `[deadline]` | `OPEN`, `TOKEN=0` | `OPEN`, `TOKEN=1`, `REG=1` | +1 if `JOBS=0` | | `pipe_reply` attach `[deadline]` | `OPEN`, `TOKEN=1` | `OPEN`, `JOBS+1` | +1 if `JOBS>=1` (the first job shares the token's credit) | | linked job ends | any, `JOBS>=1` | same state, `JOBS-1` | -1 if this lowers `credits` | -| reply | `OPEN` | `WRITING`, then `REPLIED` or `TIMED_OUT` | -1 if `TOKEN` and `JOBS=0` | +| reply | `OPEN` | `REPLIED` (no clock read) | -1 if `TOKEN` and `JOBS=0` | | drop, overloaded, expiry, abandonment, registry close | `OPEN` | `DROPPED` / `OVERLOADED` / `TIMED_OUT` / `ABANDONED` / `STOPPED` | -1 if `TOKEN` and `JOBS=0` | | `is_closed()` `[deadline]` | `OPEN`, past deadline | `TIMED_OUT` | as expiry | Rules: -1. **Reply.** The reply is `CAS OPEN -> WRITING`, which keeps the - `TOKEN`/`REG`/`JOBS` bits. It then reads `deadline` and `now`: - - at or past the deadline: CAS the state field `WRITING -> TIMED_OUT`, and - return the value as `Expired`; - - otherwise: write the value, then CAS the state field - `WRITING -> REPLIED`. - - These CASes change only the state field and retry on concurrent `JOBS` - changes; a whole-word store would lose a job-end decrement. - - Only `REPLIED` publishes the value; a receiver that sees `WRITING` waits. - A registry close that finds `WRITING` waits for it to resolve. The window - is a value write; no user code runs in it. -2. **Shortening.** `with_deadline(at)` is `fetch_min(deadline, at)`, then the - `[deadline]` check. The linearization point is the reply's `WRITING` CAS: - - a shortening before it is seen by the reply; - - a shortening after it finds the word not `OPEN` and does nothing. +1. **Reply.** The reply writes the value under the slot lock and publishes + it with one `CAS OPEN -> REPLIED` (keeping the `TOKEN`/`REG`/`JOBS` + bits; it retries on concurrent `JOBS` changes). A deferred reply holds + the owner's reply permit across that CAS (the death-point fence). The + reply reads no clock: the caller's expiry is the only timeout authority, as in OTP's +`gen_server:call`: a reply reads no clock. A reply that loses the race with +the expiry (the caller's timer, the owner's deadline heap, or a +`[deadline]` check) is discarded (`Delivery::Expired`) and the caller sees +`RequestTimeout`; a reply that lands after the deadline instant but before +the expiry runs is delivered (`Delivery::Delivered`, and the caller gets +the value), as an OTP reply already in the caller's mailbox when its +`after` clause would fire is received. A reply that finds the + word not `OPEN` takes its value back (`Expired`, `CallerGone`, + `OwnerExited`). +2. **Shortening.** `shorten_deadline(at)` lowers the deadline under the slot + lock while the word is `OPEN`, then runs the `[deadline]` check: a new + deadline already passed expires the slot at once, racing any reply on + the word (exactly one wins; a shortening after the reply does nothing). 3. **Registry.** The registry is a per-instance list of weak slot references, appended by `defer()` after its CAS and pruned lazily; the `REG` bit in the word is authoritative. Closing the registry CASes every listed slot that is @@ -172,15 +204,15 @@ There is one timeout reason, as in OTP. `DeadlineExceeded` exists only on | Today | Proposed | |---|---| -| `request_raw(msg) -> Result, _>`, no timeout | `request_raw(msg) -> Result, _>`, default 5 s deadline (as `gen_server:call/2`); `request_raw_until(msg, Deadline)` with `Deadline::At(t)` or `Deadline::Never` | -| `Receiver::request_raw` | `request_raw_until(msg, Deadline)` (required) plus a provided `request_raw` | -| erased `request(recipient, msg, timeout)` | computes the deadline before enqueueing, then `request_raw_until` | +| `request_raw(msg) -> Result, _>`, no timeout | `send_request(msg, impl Into) -> Result, _>` (OTP's `gen_server:send_request/2` with an explicit deadline): a `Duration`, an `Instant` or `Deadline::Never` | +| `Receiver::request_raw` | `send_request(msg, Deadline)` (required) | +| erased `request(recipient, msg, timeout)` | computes the deadline before enqueueing, then `send_request` | | `Response::from(Result, _>)`, `Response::from_with_timeout` | `Response::from(Result, _>)`; `from_with_timeout` removed (the deadline is already in the slot) | -| `#[protocol]` tasks code: `from_with_timeout(target.request_raw(m), t)` | the timeout is chosen first (per-call override, method, protocol, default), then `Response::from(target.request_raw_until(m, now + t))` | +| `#[protocol]` tasks code: `from_with_timeout(target.request_raw(m), t)` | the timeout is chosen first (per-call override, method, protocol, default), then `Response::from(target.send_request(m, Deadline::after(t)))` | | `#[protocol]` threads code: `Response::ready(target.request_with_timeout(m, t))` | unchanged: threads `Response` stays eager and blocks in the call | | `ActorError`, exhaustive | `#[non_exhaustive]`, plus `ReplyDropped` and `Overloaded` | | `metrics::Snapshot` request counters | add `requests_reply_dropped_total` and `requests_overloaded_total` (`RequestObservation::finish`) | -| `spawned-test` `Mock`, `Probe` | `Mock` implements the new `Handler` (below); `Probe` implements `request_raw_until` and discharges unexpected requests as `Stopped`, keeping its contract | +| `spawned-test` `Mock`, `Probe` | `Mock` implements the new `Handler` (below); `Probe` implements `send_request` and discharges unexpected requests as `Stopped`, keeping its contract | `ReplyReceiver` is a future in `tasks` mode and has `recv()` in `threads` mode. Dropping it is `CAS OPEN -> ABANDONED`, which returns the token's credit @@ -196,7 +228,7 @@ then expires the slot; there is one deadline, not two timers. The behaviour changes in two places, both in the migration notes: - a `#[protocol]` request's timeout used to start at first poll; -- `request_raw` used to have no timeout. +- `request_raw` used to have no timeout (`send_request` takes one explicitly). We use send time because the deadline is a property of the request, the server cannot learn when the caller polls, propagated deadlines (rank 19) need an @@ -204,14 +236,20 @@ absolute instant, and OTP's `call` starts waiting when it sends. **Who expires a slot.** -- **A polling receiver.** It sleeps until the slot's deadline (rt clock) and - then `CAS OPEN -> TIMED_OUT`. +- **A polling receiver.** `request`, `request_with_timeout` and + `Response::from` sleep on the runtime's timer until the slot's deadline + and then `CAS OPEN -> TIMED_OUT` (so they follow paused Tokio clocks). A + raw receiver awaited directly (`send_request(..)?.await`) instead puts its + deadline once in the owner's heap, which expires it on the service's clock + (real time, or the simulated clock under `spawned_sim`), saving a timer + per request. A shortened deadline (`ReplyTo::shorten_deadline`) wakes the + awaiting or parked receiver, which re-arms for it. - **`defer()`, the `pipe_reply` attach and `is_closed()`.** They check the deadline first, so an unpolled, expired request reports `Closed`. - **The owner's deadline heap,** for deferred tokens and for job deadlines. It returns credits and delivers job deadline notifications even when no caller is polling. -- **A reply itself:** at or past the deadline it expires the slot (rule 2). +- **Not the reply:** a reply reads no clock (rule 1, as OTP). A queued or in-turn request that nobody polls stays open until it is polled, replied (and expired), deferred (`defer()` on an expired slot returns @@ -229,10 +267,12 @@ handles a call even after the caller timed out, and the late reply is dropped. - **Stale entries.** An entry is stale when its slot or job is terminal, or when its deadline is later than the current one (superseded by - `with_deadline`). + `shorten_deadline`). - **Compaction.** The heap is rebuilt when stale entries exceed live ones, per actor. So storage is at most about 2 ร— (live deferred tokens + live - jobs with deadlines), at most about 2 ร— (`MAX_OUTSTANDING` + job count). + jobs with deadlines + awaited raw receivers); the first two are bounded by + `MAX_OUTSTANDING` + job count, the last by the requests callers are + awaiting (each also holds a queued envelope or an open slot). - **Ownership.** Entries hold `Weak` references, so the heap owns no slot and no job. The heap is dropped at the owner's death, after the registry close. - **Service.** A per-system service holds one entry per actor: its earliest @@ -296,6 +336,7 @@ impl PeerTable { Err(DeferError::Overloaded) => {} // caller already answered Overloaded Err(DeferError::Closed) => {} // caller already timed out or left Err(DeferError::NoCaller) => {} // reached by send + Err(_) => {} // #[non_exhaustive] } } } @@ -309,7 +350,7 @@ pub struct PendingReply<'t, R> { .. } impl PendingReply<'_, R> { pub fn reply(self, value: R) -> Delivery; // Delivered | CallerGone | Expired pub fn defer(self) -> Result, DeferError>; // Overloaded | Closed | NoCaller - pub fn deadline(&self) -> Option; + pub fn deadline(&self) -> Deadline; // Never for sends pub fn is_closed(&self) -> bool; // checks the deadline first pub fn overloaded(self); // answer Err(Overloaded) } @@ -318,8 +359,8 @@ impl PendingReply<'_, R> { pub struct ReplyTo { .. } // Send + 'static, not Clone impl ReplyTo { pub fn reply(self, value: R) -> Delivery; // .. | OwnerExited - pub fn deadline(&self) -> Option; - pub fn with_deadline(&self, at: Instant); // shorten only + pub fn deadline(&self) -> Deadline; + pub fn shorten_deadline(&self, d: impl Into); // shorten only pub fn is_closed(&self) -> bool; pub async fn closed(&self); // tasks mode pub fn owner(&self) -> ActorId; @@ -345,6 +386,13 @@ No rule uses `thread::panicking()`. A drop that lands on the turn list from another task while a turn runs is committed at that turn's end. The delay is at most one turn, and the outcome only depends on whether that turn panicked. +**Refined by rank 5e** ([otp-exit-signals.md](otp-exit-signals.md#required-changes-to-rank-11-221-design-239-implementation), +implemented in #240): a `Stopped` outcome is no longer answered at once. The +slot becomes *exit pending*: it stays open in the registry and the death +point answers it `ActorStopped` with the owner's final exit reason (or the +caller's deadline comes first). An undispatched queued request is held the +same way, and a send or request after the death point reports `NoProc`. + ## 5. Owner lifecycle Each actor instance has a reply registry (its open deferred tokens and the @@ -367,14 +415,38 @@ current turn's slot) and a job set. The order is: answered. 3. **Death.** This happens only when `stopped()` has returned. Rust cannot preempt `stopped(&mut self)` or a hanging destructor, and it cannot - destroy state that is still borrowed. In this order: - 1. the registry closes, and every open token it lists becomes `STOPPED`, + destroy state that is still borrowed. In this order, whether the run + loop or the runtime dropping the actor drives it: + 1. job admission closes and every job is cancelled; outgoing replies of + deferred tokens and exit signals sent through the actor's contexts + are fenced (those in flight finish, later ones are refused: a reply + returns `OwnerExited`, a signal is inert); + 2. the final reason is fixed, the link table closes and admission is + sealed (`NoProc`); + 3. the registry closes, and every open token it lists becomes `STOPPED`, wherever the token is (state, job, helper, leaked); - 2. the mailbox is disposed: queued envelopes are dropped as `STOPPED`, and - emit credits return; - 3. actor state is dropped; - 4. the deadline heap is dropped; - 5. the exit reason is published, and monitors and links fire. + 4. the mailbox closes and every queued request is answered `STOPPED`; + 5. only then are payloads destroyed: queued envelopes (emit credits + return), then actor state; + 6. the deadline heap is dropped; + 7. the exit reason is published, and monitors and links fire. + + Steps 1 to 4 run no user code: steps 3 and 4 only make terminal + transitions, and every slot they meet stays referenced (an unread + reply's value is user code when destroyed) until step 5. After the last + answer, and before step 5, run in this order: the parked blocking + receivers are notified (no user code, so none waits behind a waker), the + callers' wakers (executor or user code), a runtime drop's "stopped() + skipped" log, and the callbacks the jobs' cancellation owes (`cancelled()` + waiters, blocked emitters). Cancellation in step 1 only sets the jobs' + flag: the payloads it releases (held results, scheduled emits) are + destroyed in step 5. From the start of the exit (the jobs' cancellation) + to the end of discharge, every payload the framework drops for the owner + on any thread (a scheduled emit coming due, a job's refused outcome) is + held by the owner's drop sink and destroyed then too. Steps 3 and 4 follow the seal instead of being one atomic step + with it, but nothing between them depends on the requests they answer, + so the models' atomic death is a sound abstraction of them (they commute + left to the seal). `ActorStopped` keeps one meaning, "the actor is gone". An observer that sees `Down` knows every caller of the instance has been answered. @@ -425,6 +497,12 @@ the actor's job set; it is not detached. non-exhaustive enum; - a request to an actor that has already died reports `NoProc`. +Both are implemented in #240 as `ActorError::ActorStopped(ExitReason)` and +`ActorError::NoProc`, with the death boundary of +[otp-exit-signals.md](otp-exit-signals.md#phases-of-an-exit): the registry is +sealed with the final reason at the death point, and exit errors wait for +it. + The slot's terminal kinds are defined so that rank 5e only adds the reason. ## 6. Delegation @@ -433,11 +511,12 @@ There is no `ReplyTo::forward` in v1. ```rust let token = reply.defer()?; -ctx.pipe_reply(token, other.request(msg), |res, token| Delegated { res, token })?; +ctx.pipe_reply(JobOptions::new(), token, |_| other.request(msg), |res, token| Delegated { res, token })?; ``` The downstream request is an ordinary request with the destination's own -admission and deadline; it should use `token.deadline()`, which rank 19 +admission and deadline; it should pass `token.deadline()` on +(`other.send_request(msg, token.deadline())`), which rank 19 automates. The obligation and credit stay in this actor's registry. When the linked job ends or is cancelled, its future, and with it the downstream `ReplyReceiver`, is dropped. That abandons the downstream slot and returns @@ -450,22 +529,30 @@ impl PendingReplies { pub fn with_capacity(cap: usize) -> Self; /// Purges terminal tokens, then inserts; a duplicate key or a full table /// hands the new token back. - pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>>; + pub fn insert(&mut self, key: K, token: ReplyTo) -> Result<(), InsertError>; pub fn take(&mut self, key: &K) -> Option>; // skips (and removes) terminal entries - pub fn purge(&mut self) -> usize; - pub fn retained(&self) -> usize; // entries held, without purging - pub fn live(&self) -> usize; // non-terminal entries (scan) + pub fn len(&self) -> usize; // live entries (scan) + pub fn is_empty(&self) -> bool; + pub fn drain(&mut self) -> impl Iterator)>; // live tokens, oldest first } +#[non_exhaustive] +pub enum InsertError { Duplicate(ReplyTo), Full(ReplyTo) } // kind(), into_inner(), into_parts() ``` Retained entries never exceed `cap`, however fast tokens expire. The handler decides what to answer on `InsertError` (typically `Overloaded` or a domain -error) and still owns the token. +error) and still owns the token. Entries keep insertion order, so `drain` +answers callers in the same order in every process: the order never comes +from a per-process hash seed, and a same-seed simulator run reproduces it +(#261; `reply::pending::tests::drain_follows_insertion_order`, +`tests/sim.rs::pending_replies_drain_is_reproducible`). The table is a scan +bounded by `cap`: `insert` already purges it whole, and `take` is linear. ## 8. Bounds **Credits per actor instance: `MAX_OUTSTANDING`** (an `Actor` constant, -default 1024, overridable per `ChildSpec`). This is a count limit, not a +default 1024; `#[actor(max_outstanding = N)]`, overridable per `ChildSpec` +with `ChildSpec::max_outstanding`). This is a count limit, not a memory budget; byte credits are rank 18. - An in-place reply needs no credit. @@ -491,7 +578,8 @@ matter which job finishes first: the request holds `max(running jobs, live token)` credits. **Native execution budget: `MAX_NATIVE_JOBS`** (an `Actor` constant, -overridable per `ChildSpec`; provisional default +`#[actor(max_native_jobs = N)]`, overridable per `ChildSpec` with +`ChildSpec::max_native_jobs`; provisional default `min(4, available_parallelism)`). It covers `threads`-mode jobs and `pipe_blocking*`. @@ -509,34 +597,44 @@ otherwise `PipeReplyError::WrongOwner` hands the token back. ## 9. Pipe-to-self jobs ```rust -pub struct JobOptions { - pub deadline: Option, // none: no job deadline - pub key: Option, // cancel-previous - pub cancel_on_caller_gone: bool, // pipe_reply only; default false (OTP) - pub emit_credits: u16, // default 16 +#[non_exhaustive] +pub struct JobOptions { /* private */ } // OTP-style option list, passed first +impl JobOptions { + pub fn new() -> Self; // = default() + pub fn key(self, key: impl Into) -> Self; // cancel-previous + pub fn deadline(self, d: impl Into) -> Self; // Duration, Instant or Deadline + pub fn emit_credits(self, n: NonZeroU16) -> Self; // default 16 + pub fn cancel_on_caller_gone(self) -> Self; // pipe_reply* only; off by default (OTP) } -// Spawn at once; options are passed first, never chained afterwards. -ctx.pipe(work, done) -> Result -ctx.pipe_with(opts, work, done) -> Result -ctx.pipe_blocking_with(opts, |job| .., done) -> Result -ctx.pipe_reply(token, work, done) -> Result> -ctx.pipe_reply_with(opts, token, work, done) -> Result> -ctx.pipe_reply_blocking_with(opts, token, |job| .., done) -> Result> - -pub enum PipeErrorKind { Overloaded, NativeBudget, SpawnFailed(io::Error), OwnerStopping } -pub struct PipeError { pub kind: PipeErrorKind } +// One call per shape; spawn at once. `work` receives the job's `JobCtx`. +ctx.pipe(opts, |job| async { .. }, done) -> Result +ctx.pipe_blocking(opts, |job| .., done) -> Result +ctx.pipe_reply(opts, token, |job| async { .. }, done) -> Result> +ctx.pipe_reply_blocking(opts, token, |job| .., done) -> Result> +// Reserve-then-spawn: the admission checks (owner running, credit, native +// slot) before `work`/`done` exist; spawning from the permit cannot fail. +ctx.reserve_job(opts) -> Result, PipeError> // .pipe(work, done) +ctx.reserve_blocking_job(opts) -> Result, PipeError> // .pipe_blocking(work, done) + +#[non_exhaustive] +pub enum PipeError { Overloaded, NativeBudget, SpawnFailed(io::Error), OwnerStopping } +#[non_exhaustive] pub enum PipeReplyError { - Failed { token: ReplyTo, kind: PipeErrorKind }, + Failed { token: ReplyTo, error: PipeError }, Closed(ReplyTo), WrongOwner(ReplyTo), } -pub enum JobOutcome { Ok(T), Err(JobError) } +pub type JobOutcome = Result; +#[non_exhaustive] pub enum JobError { Panicked(String), DeadlineExceeded } impl JobCtx { pub fn is_cancelled(&self) -> bool; pub async fn cancelled(&self); pub async fn emit(&self, msg: M) -> Result<(), JobGone> where A: Handler; pub fn emit_blocking(&self, msg: M) -> Result<(), JobGone> where A: Handler; + // Delayed emit: credit taken now, step 2 at `at` (see the emit protocol). + pub async fn emit_at(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; + pub fn emit_at_blocking(&self, at: Instant, msg: M) -> Result<(), JobGone> where A: Handler; } ``` @@ -589,6 +687,20 @@ handle, the key map and every message it produces. It has three parts: which return `JobGone`. A blocked emitter therefore never holds up the stop-time join. +**Delayed emits (`emit_at`).** Step 1 runs at the call. A scheduled emit is +then registered under the job's lock (refused, `JobGone`, if the job is no +longer accepting emits) and serviced by the owner's deadline heap; at its +instant, step 2 runs as for `emit`. So it is fenced and credit-accounted like +any emit. To keep the terminal message last, the work's result is **held** +while scheduled emits remain: the last one to fire enqueues it. Cancellation +drops the scheduled emits and a held result; a job deadline takes the +notification as usual (deadlines are hard), and later scheduled emits are +then dropped at their instant. Modelled in `DeferredJobs.tla` +(`MCDeferredJobsHeld`: scheduled emits, the held result, the deadline taking +it and the job's credit kept until the terminal message; controls +`EarlyCredit` and `HeldDeadlineLost`); Loom model +`scheduled_emit_vs_finish_and_cancel`. + **Fencing at dispatch.** Every job message carries the job's `JobState` and the owner's `ActorId`. The run loop drops it before the handler runs if: @@ -596,7 +708,7 @@ the owner's `ActorId`. The run loop drops it before the handler runs if: `ActorId`, not a per-slot supervisor generation); or - the job is cancelled. -**Keyed replacement.** `pipe_keyed` / `JobOptions::key` cancels the previous +**Keyed replacement.** `JobOptions::key` cancels the previous job for the key **whatever its notification state**: pending, notified (its deadline fired) or finished with a queued terminal message. So none of its queued messages reach the handler. The key map holds only the current job. @@ -612,11 +724,14 @@ panic, so no consumed `FnOnce` has to be retried. **Executors.** -- **`tasks` mode:** async jobs on the owner's runtime and blocking jobs on the - native budget. +- **`tasks` mode:** async jobs on the owner's runtime; blocking jobs on + Tokio's blocking pool (`spawn_blocking` on the managed actor's group + runtime, or the current one), under the native budget. Outside a Tokio + runtime, and under the simulator, an owned thread. - **`threads` mode:** one owned native thread per admitted job under the - native budget. Spawning is fallible (`SpawnFailed`). The shared pool comes - in rank 11b. + native budget. Spawning is fallible (`SpawnFailed`); the thread is spawned + at reservation and parked until its job starts, so a permit's spawn is not. + A managed compute pool comes in rank 11b. - **Unmanaged actors** own their job set like managed ones; managed actors also register theirs with the scope. - **Jobs must not wait on their own owner.** Once the owner is exiting it no @@ -698,7 +813,7 @@ Aligned with OTP, so not differences: - **`stopped()` may reply**, and callers learn at death. - **Replies are immediate.** - **Completions are turns** (`handle_info`). -- **`pipe_keyed` is shorthand** for cancel then start. +- **A keyed job (`JobOptions::key`) is shorthand** for cancel then start. ## Decisions on rank 11a @@ -715,7 +830,7 @@ Aligned with OTP, so not differences: | Rank | Reuses | |---|---| -| 11b | `JobState`, the emit protocol, fencing, `done`, credits, `PipeErrorKind` | +| 11b | `JobState`, the emit protocol, fencing, `done`, credits, `PipeError` | | 13 | `stopped()` replies, the registry outstanding count, the unfinished report | | 15 | the per-instance job set, keyed cancel-previous, the bounded join | | 15a | a slot is a request scope; `JobOptions::deadline` is never later than its parent's | @@ -739,16 +854,16 @@ Aligned with OTP, so not differences: **Timeouts.** - The request timeout starts at send, not at first poll. -- `request_raw` gains a 5 s default; use `request_raw_until(msg, - Deadline::Never)` for the old unbounded wait. +- `request_raw(msg)` becomes `send_request(msg, deadline)`; 0.7's + unbounded wait is `send_request(msg, Deadline::Never)`. **Raw API.** -- `request_raw` returns `ReplyReceiver`, which yields +- `send_request` returns `ReplyReceiver`, which yields `Result`. - `Response::from_with_timeout` is removed. - `Receiver` implementors (including `Probe`) implement - `request_raw_until`. + `send_request`. **Lifecycle.** @@ -800,8 +915,10 @@ It establishes, within its bounds: - **Discharge.** At most one discharge per request (`AtMostOneDischarge`), and no open slot without a live holder, including job-held tokens at death (`NoOrphan`). -- **Deadlines.** No success at or after the deadline - (`NoSuccessAfterDeadline`); deadlines are serviced while a turn is stuck +- **Deadlines.** Only the expiry decides a timeout; no success after the + slot expired, so none after the deadline instant + (`NoSuccessAfterDeadline`: `okAt <= dl`; negative control `LateSuccess`, a + reply overriding an expired slot); deadlines are serviced while a turn is stuck (`TimelyExpiry`); shortening never extends past the caller's deadline (`DeadlineWithinCaller`). - **Credits.** Combined token and linked-job accounting with attach-on-open @@ -815,6 +932,13 @@ It establishes, within its bounds: owner (cleanup deadline passed while `stopped()` runs) closes nothing early: no token is answered `Stopped` before death (`NoEarlyClose`). `stopped()` may still reply while stuck, and death may never follow. +- **Death boundary.** Death is the three steps of `actor_cleanup.rs`: the + death point fences the reply gate and seals the registry, then the batch + answers every held token and queued request one at a time (callers and the + deadline service interleave), then the collected wake-ups run. No reply is + admitted after the death point (`NoReplyAfterDeath`, control + `NoReplyFence`), and a caller woken with `Stopped` finds the whole death + answered (`WokenAfterDischarge`, control `CallbackBeforeDischarge`). - **Downstream cleanup.** A downstream request ends with its last job (`DownstreamOwned`). - **Liveness without handler fairness:** `EveryAcceptedDischarged` and @@ -837,6 +961,12 @@ It establishes: - **Terminal messages.** `AtMostOneTerminal` and `TerminalLast`. - **Bounds.** `PermitsBounded` (running work keeps its permit), `OneLivePerKey` and `EmitCreditsBounded`. +- **Held results.** A held result keeps its credit until its terminal + message (`HeldCounted`) and its deadline stays armed + (`DeadlineArmedWhileHeld`). +- **Owner exit.** Admission and `cancel_all` are serialized, and the exit + cancels running work, scheduled emits and held results alike + (`ExitCancelsJobs`, controls `NoAdmissionFence` and `ExitKeepsHeld`). - **Liveness.** `JobReported` for jobs with a deadline. Abstraction: credits return at dispatch (fenced or not); mailbox disposal at @@ -855,7 +985,8 @@ Requests and timing: Tokens and synchronization: - token drops by other tasks during an unrelated turn; -- `WRITING` versus registry close; +- `WRITING` versus registry close (the reply permit is modelled as the + reply gate closing at the death point; the CAS race is Loom territory); - the packed-word counter deltas; - memory orderings and wakeups. @@ -922,6 +1053,41 @@ ordering and `EmitCreditsBounded` checks in-flight emit credits. **Trace validation:** against both models. +## Implementation notes + +Mechanism choices made by the implementation (PR #239). None changes the +semantics above. + +- **Value cell.** The crates are `#![forbid(unsafe_code)]`, so the slot's value + and the caller's waker live under one uncontended `Mutex` instead of an + `UnsafeCell` plus `AtomicWaker`. The packed word, its CAS rules and every + credit delta are as specified; the reply reads the deadline under the same + lock that a shortening takes, which makes a shortening after the `WRITING` + CAS a no-op (Loom: `reply_vs_shortening`). +- **Receiver and timer.** `ReplyReceiver` is one pointer and implements + `IntoFuture`; the deadline timer lives in the awaiting `ReplyFuture`. On + the `request` and `Response` paths, when the runtime's timer fires the + receiver expires the slot, so paused Tokio clocks keep working. A raw + receiver awaited directly registers its deadline in the owner's heap + instead (no runtime timer per request), so it follows the deadline + service's clock, not a paused Tokio clock. +- **Deadline service.** Unmanaged and managed actors share one process-wide + service thread (not a task on the `ActorSystem` control runtime); a dead + owner is forgotten at its registry close. Under `spawned_sim` each owner's + heap is served by a task of the simulated runtime, so clock domains never + mix. +- **Cleanup deadline.** Armed when exit begins: the supervisor's + `shutdown_timeout` for a supervised child, the remaining system grace + during `ActorSystem::shutdown`, otherwise `DEFAULT_STOP_DEADLINE` (5 s). + Jobs still running then appear in `ShutdownReport::unfinished_jobs` and as + an `unfinished_at_stop` event. +- **Async emits.** Every `pipe*` hands the work its `JobCtx` (`emit`, + `cancelled`): `ctx.pipe(opts, |job| async move { .. }, done)`; `|_| fut` + when the work needs neither. +- **Destruction.** If an owner's lifecycle future is dropped (its runtime + destroyed), exit begins before its state is dropped and the registry + closes, so open tokens answer `ActorStopped`. + ## Open questions 1. Exact names (`PendingReply`, `DeferError`, `PipeReplyError`) and whether @@ -961,7 +1127,7 @@ changes). | 2.1.1 Read before publish | `WRITING` state; `REPLIED` stored after the value (ยง1). | | 2.1.2 Permit install racing a terminal slot | `TOKEN` bit set by a CAS that requires `OPEN` (ยง4). | | 2.1.3 Undispatched envelope drop | Mailbox disposal discharges `STOPPED` (ยง5 step 3.2); only a turn's `PendingReply` drop gives `Dropped`. | -| 2.2.1 `request_raw` gains a timeout | Stated; `Deadline::Never` keeps the unbounded wait (ยง1, migration). | +| 2.2.1 `request_raw` gains a timeout | `send_request(msg, deadline)` takes it explicitly; `Deadline::Never` keeps the unbounded wait (ยง1, migration). | | 2.2.2 Stale doc comments in code | Implementation task: `macros/src/lib.rs` "timers start on poll" and `tasks/actor.rs` request docs. | | 2.3.1 Unpolled expired requests dispatched | Dispatched by design (OTP); `defer()` on an expired slot returns `Closed`; skipping is opt-in (rank 19). | | 2.3.2 Process-wide service on the first runtime | Dedicated service thread for unmanaged actors; per-sim-instance under sim (ยง2). | @@ -972,14 +1138,14 @@ changes). | 2.5.1 Attach after expiry | Attach CAS requires `OPEN`; otherwise `Closed` with the token (ยง8). | | 2.5.2 `TimedOut` does not trigger cancellation | The opt-in coupling triggers on `ABANDONED` and `TIMED_OUT` (ยง9). | | 2.5.3 `WrongOwner` shape; token lost on error | `PipeReplyError` returns the token in every variant (ยง9). | -| 2.5.4 No `pipe_reply_with` | `pipe_reply_with` and `pipe_reply_blocking_with` added (ยง9). | +| 2.5.4 No `pipe_reply_with` | `pipe_reply` and `pipe_reply_blocking` take `JobOptions` first (ยง9). | | 2.6.1 Keyed replacement after a deadline | Fixed (Codex 4). | | 2.6.2 Fenced payload drop panics | `DropFailures` path (ยง1 rule 5). | | 2.7.1 Emit credit deadlock on stop | Cancel and exit wake blocked emitters with `JobGone`; credits also return at mailbox disposal (ยง9). | | 2.7.2 Emits after the deadline | Emit check and terminal enqueue are serialized under the job lock (ยง9). | | 2.8 `R: From`, local-slot deadlock | Removed; explicit trait migration (ยง3). | | 2.9.1 Threads `Response` stays eager | Stated (ยง1 API table). | -| 2.9.2 `Probe` | Implements `request_raw_until`; unexpected requests answer `Stopped` (ยง1). | +| 2.9.2 `Probe` | Implements `send_request`; unexpected requests answer `Stopped` (ยง1). | | 2.9.3 `PendingReplies::len(&self)` | Replaced by non-mutating `retained` and `live`; purging is in `&mut` methods (ยง7). | | 2.9.4 Metrics type name | `metrics::Snapshot` and `RequestObservation::finish` (ยง1). | | 3.A model divergences | `HeldA` replaced by `Credits` with two holders; expiry abstraction stated; `pipe_reply` notification is a turn (`JobNotify`); cancellation is opt-in (`JobCancel` any closed slot). | diff --git a/docs/design/otp-exit-signals.md b/docs/design/otp-exit-signals.md index eac930398..e9c22e2ae 100644 --- a/docs/design/otp-exit-signals.md +++ b/docs/design/otp-exit-signals.md @@ -6,8 +6,12 @@ maintainer settled every open question (see [Decisions](#decisions)). - Covers ROADMAP rank 5e (exit signals), the startup part of rank 5f, rank 5g (runtime-dropped actors) and the supervisor `start()` ordering found by the - quality pass (#223). Counting the restart budget per recovery round (the - other half of 5f) is a separate change. + quality pass (#223). The other half of 5f, counting the restart budget per + recovery round, shipped in the same 0.8 stack: a OneForAll/RestForOne + round takes one grant when the coordinator selects it (OTP's + `add_restart` once per restart action), its rebuilds take none, and a + failed replacement requests a new round that is charged again (semantics + V5). - Target: v0.8. Changes L3, L4, L6, L7, X1-X7, V3, V7, V9 and ยง11 of the [semantics contract](semantics.md), and refines the death boundary of [deferred-replies.md ยง5](deferred-replies.md#5-owner-lifecycle) (rank 11, @@ -33,7 +37,7 @@ maintainer settled every open question (see [Decisions](#decisions)). | E3 | a trapping gen_server that gets `'EXIT'` from its parent runs `terminate(Reason)` | supervisor shutdown cancels a trapping child directly; a `start_linked` parent's exit reaches `exit_received` | a signal from the actor's parent is queued and, when dispatched, runs normal termination; it never reaches `exit_received` | | E4 | `link`/`monitor` on a process that no longer exists give `noproc` | the target's real exit reason | `NoProc` after D | | E5 | exit reasons are any term | closed four-variant enum | `#[non_exhaustive]`, adds `Killed` and `NoProc` | -| E6 | `unlink`/`demonitor` stop every later effect of the edge | a notice already taken for delivery still arrives | fenced at the receiver; only a message already queued stays | +| E6 | `unlink`/`demonitor` stop every later effect of the edge (`demonitor(Ref, [flush])` also removes a queued `'DOWN'`) | a notice already taken for delivery still arrives | fenced at the receiver; an `Exit` already queued stays, a queued `Down` is flushed | | E7 | signals between a pair of processes keep their order | death notices follow earlier messages | unchanged for death notices; direct signals are per-sender FIFO with messages | | S1 | `start_link` aborts on any child start error; exits of started children wait until `init` returns | Permanent/Transient children retry a failed first start; a ready child's exit during startup is handled at once | any first-start failure aborts; ready children's exits are processed after the initial start completes | | S2 | `start_link` returns its error after the failed child exited and started children were terminated | `start()` can return `Err` before the failed child is `Failed` | `start()` returns `Err` after rollback (or its deadlines) | @@ -510,8 +514,13 @@ signal in flight"). pending, removes it, and enqueues the `Exit`/`Down` (or, for a non-trapping receiver, requests its exit) **inside that critical section**. - `unlink` and `demonitor` remove the record under the same lock, live or - pending. After they return nothing of that edge takes effect; a message + pending. After they return nothing of that edge takes effect; an `Exit` already queued stays. +- **Flush (0.8).** A monitor's record outlives its delivery: delivery marks + it *queued*, and the `Down`'s dispatch takes it (a private envelope, so no + user-sent `Down` is affected). `demonitor` removes a queued record too, so + the dispatch then skips the `Down`, as `demonitor(Ref, [flush])`. Ordinary + messages pay nothing; a `Down` pays one table lock at dispatch. - A late `monitor` or late `link` leaves a pending record for its `NoProc` notice, so `demonitor` or `unlink` cancels it, as OTP keeps the caller's link record while the `noproc` response is pending. @@ -764,7 +773,7 @@ uses `stop_with_reason`, `Kill`, `is_abnormal`, `exit_reason()` or |---|---|---| | E1 Killed | none | none | | E2 exit signals | none, because `Context::stop` stays a self-exit. Two sites call `stop()` on a `Context` clone from outside the handler: [`rlpx/connection/server.rs:1162`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L1162) (a stream listener stopping its own actor) and the test [`shutdown_order_tests.rs:197`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/test/tests/p2p/shutdown_order_tests.rs#L197) (`table.actor_ref().context().stop()` on a mock, then `join`) | none: [`http_server.rs:87`](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/bin/ethlambda/src/http_server.rs#L87) stops its own actor from a helper so the Permanent supervisor restarts it | -| E3 parent exit | none (no trapping actors; supervised and system-owned actors stop as today) | none | +| E3 parent exit | **behaviour change**: no actor traps exits, so a supervised or system-owned actor stopped by supervisor or system shutdown now ends `Shutdown` **without `stopped()`** (decision 2, as OTP skips `terminate` for a non-trapping gen_server); an actor that needs `stopped()` for cleanup on shutdown must call `ctx.trap_exit(true)` in `started()`. The two ethrex actors with a `stopped()` ([`rlpx/connection/server.rs:624`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/rlpx/connection/server.rs#L624), `tooling/monitor/src/app.rs:126`) are started with `.start()` (no parent) and are unaffected | none (no actor defines `stopped()`) | | E4 NoProc, E6 fences | none | none | | E5 non_exhaustive | no break ([`discovery/supervised.rs:261`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/discovery/supervised.rs#L261) uses `matches!`); `shutdown.rs:170` logs with `Display` | no break ([`http_server.rs:242`](https://github.com/lambdaclass/ethlambda/blob/6a12bc753ae18c9d63f3dbf930edc700c4c4704c/bin/ethlambda/src/http_server.rs#L242)) | | S1 startup aborts | **behaviour change**: the Transient discovery child's factory [`DiscoveryServer::build`](https://github.com/lambdaclass/ethrex/blob/0dd4c105f65d544b0fb5e92d8634a1699127f826/crates/networking/p2p/discovery/supervised.rs#L145) can fail at boot (a UDP send error pinging a bootnode); v0.7 retries it, v0.8 fails `start()` and `init_network` panics on `.expect("Network starts")`. Fix: make the bootnode ping best-effort | none: the HTTP child's factory is infallible; a bind failure happens after readiness and is a normal restart | diff --git a/docs/design/semantics.md b/docs/design/semantics.md index 624900a91..e253b6f69 100644 --- a/docs/design/semantics.md +++ b/docs/design/semantics.md @@ -76,20 +76,25 @@ duplicated or retried. While the actor keeps running it is handled exactly once. Checked by: [`actor_scenarios`][actor_scenarios], [`payload_stress`][payload_stress] (FIFO delivery or complete destruction), [`request_histories`][request_histories]. -**D3. Accepted is not handled.** `send` returns `Err(ActorStopped)` only once -the mailbox is closed; the rejected message is dropped. `Ok(())` means -accepted: between a stop request and mailbox closure (including during -`stopped()`), sends still succeed and the message is then discarded. +**D3. Accepted is not handled.** `send` returns `Err(NoProc)` only once the +actor is past its death point (the mailbox refuses admission); the rejected +message is dropped. `Ok(())` means accepted: between a stop request and the +death point (including during `stopped()`), sends still succeed and the +message is then discarded (a request is answered `ActorStopped` with the +final reason). Checked by: [`semantics_gaps`][semantics_gaps]`::{native,tasks}_contract::stop_window_accepts_then_discards_and_releases_before_join` (sends after the stop request and from `stopped()` are `Ok` and never handled, every placement), [`mailbox_admission_loom`][mailbox_admission_loom] (threads send gate: publish before drain or fail), -`src/mailbox.rs::accepted_sends_are_drained_even_when_senders_outlive_receiver`. +`src/mailbox.rs::accepted_sends_are_drained_even_when_senders_outlive_receiver`, +[`actor_scenarios`][actor_scenarios] ("accepted sends alone do not +guarantee execution across termination"), `death_boundary` (a request +accepted before the death point is answered at it). **D4. Stop discards the queue.** The run loop checks for stop before every receive. Once a stop is observed no further message is dispatched, including messages already queued. Queued messages are dropped one by one and every -queued reply sender is released (callers get `ActorStopped`) before the exit is -published. +queued request is answered (callers get `ActorStopped` with the final reason) +before the exit is published. Checked by: `src/tasks/actor.rs::stop_while_idle_does_not_dispatch_a_queued_request_on_resume`, `src/threads/actor.rs::repeated_stop_during_a_handler_discards_queued_requests`, `src/threads/actor.rs::stop_while_receiving_does_not_dispatch_a_later_request`, @@ -105,12 +110,21 @@ Checked by: [`semantics_gaps`][semantics_gaps]`::{native,tasks}_contract::unboun ## 3. Requests -**R1. Three outcomes.** `request`, `request_with_timeout`, protocol calls and +**R1. One terminal outcome.** Every request has a reply slot +([deferred replies](deferred-replies.md) ยง1) that is discharged exactly once. +`request`, `request_with_timeout`, protocol calls and `spawned::tasks::request` end in exactly one of `Ok(reply)`, -`Err(ActorStopped)` (not accepted, or the reply sender was dropped: the actor -stopped before replying, the handler panicked, or the request was still queued -at stop) or `Err(RequestTimeout)`. A request racing shutdown gets its reply or -`ActorStopped`; a slow teardown can still produce a timeout first. +`Err(ActorStopped(reason))` (the handler panicked, the request was still +queued at the owner's death, or a deferred token was open at death; always +the owner's final exit reason, the one `wait_exit` and `Down` report), +`Err(NoProc)` (sent after the owner's death point), +`Err(RequestTimeout)` (the deadline fixed at send passed), +`Err(ReplyDropped)` (the handler dropped the reply obligation unanswered) or +`Err(Overloaded)` (`defer()` at `MAX_OUTSTANDING`, or the handler answered +`overloaded()`). A request racing shutdown gets its reply, `ActorStopped` or +`NoProc`; a slow teardown can still produce a timeout first. `ActorStopped` is +reported only at the owner's actual death, never while `stopped()` still +runs (otp-exit-signals.md, "Exit errors wait for D"). Checked by: [`actor_scenarios`][actor_scenarios], `src/tasks/actor.rs::request_timeout`, [`timeout_outcomes`][ex_timeout_outcomes] example. @@ -119,11 +133,36 @@ only the caller that sent that request. Checked by: [`actor_scenarios::concurrent_producers_preserve_order_and_reply_identity`][actor_scenarios], [`sim_ordering::per_sender_fifo_and_reply_identity_{pct,dfs}`][sim_ordering]. -**R3. Timeouts.** Default 5 s. Precedence: `with_request_timeout` handle, -`#[request_timeout]` method attribute, `#[protocol(timeout)]`, 5 s. Send-only -methods have no timeout. tasks: `request`-family futures enqueue on first poll -and protocol methods enqueue when called; the timer starts at first poll. -threads: the call blocks and the timer starts immediately. +**R3. Timeouts.** Default 5 s, as `gen_server:call/2`; `send_request(msg, +deadline)` takes its deadline explicitly (`Deadline::Never` waits without +one). +Precedence: `with_request_timeout` handle, `#[request_timeout]` method +attribute, `#[protocol(timeout)]`, 5 s. Send-only methods have no timeout. +The deadline is fixed when the request is sent (send time plus the timeout) +and travels in the reply slot: there is one deadline, not a caller timer plus +a server one. tasks: `request`-family futures enqueue on first poll and +protocol methods enqueue when called; either way the deadline starts at the +enqueue. threads: the call blocks. Expired requests are still dispatched (OTP). +The caller's expiry is the only timeout authority, as in OTP's +`gen_server:call`: a reply reads no clock. A reply that loses the race with +the expiry (the caller's timer, the owner's deadline heap, or a +`[deadline]` check) is discarded (`Delivery::Expired`) and the caller sees +`RequestTimeout`; a reply that lands after the deadline instant but before +the expiry runs is delivered (`Delivery::Delivered`, and the caller gets +the value), as an OTP reply already in the caller's mailbox when its +`after` clause would fire is received. Timer resolution (the runtime timer's 1 ms +wheel, the deadline service's wake-up) bounds that window. +Which clock expires a waiting caller is part of the contract: `request`, +`request_with_timeout` and `Response::from(rx)` arm the runtime's timer, so +they follow paused Tokio time; a `ReplyReceiver` awaited directly +(`send_request(..)?.await`) puts its deadline in the owner's deadline heap +on its first pending poll instead, serviced on the real clock (the +simulated one under `spawned_sim`), not paused Tokio time. (The slot finds +its owner by a registry key in its word, so it holds no reference to it, +and a request answered before that poll never touches the heap.) The reason is cost: a raw receiver is the building +block for fan-out and needs no runtime timer per request. Tests that pause +Tokio time wait through `request` or `Response::from`. Either way a +shortened deadline (`ReplyTo::shorten_deadline`) wakes the waiting caller. Checked by: [`protocol_timeouts`][protocol_timeouts], [`cancellation_safety::protocol_calls_are_cancel_safe_and_send_eagerly`][cancellation_safety], [`protocol_timeouts`][ex_protocol_timeouts] example (both modes), @@ -136,11 +175,53 @@ or roll back effects. The caller cannot tell whether it ran. Checked by: [`cancellation_safety::request_family_is_cancel_safe`][cancellation_safety], [`request_histories`][request_histories] (timed-out operations are "pending"). -**R5. Handler panic.** A panic while handling a request drops its reply sender: -the caller gets an error, not a reply, and the actor exits with `Panic`. -Checked by: [`semantics_gaps`][semantics_gaps]`::{native,tasks}::panicking_request_handler_gives_the_caller_actor_stopped` -(exactly `ActorStopped`, every backend, a managed group, `Recipient` and raw -receivers), [`panic_safety`][panic_safety]. +**R5. Handler panic.** A panic while handling a request drops its reply +obligation in the panicking turn: at the turn's end it becomes exit pending +(a turn that returns without answering gives `ReplyDropped`), the actor exits +and the request is answered at its death point with the final reason, +`ActorStopped(Panic(..))` (or `ActorStopped(Killed)` if a kill arrived +before death), or by the caller's deadline first. +Checked by: [`deferred_replies`][deferred_replies] +`::{tasks_mode::panicking_turn_answers_actor_stopped, threads_mode::panic_and_death_answer_actor_stopped}`, +[`semantics_gaps`][semantics_gaps]`::{native,tasks}::panicking_request_handler_gives_the_caller_actor_stopped` +(exactly `ActorStopped(Panic(_))`, every backend, a managed group, +`Recipient` and raw receivers), [`otp_exit_signals`][otp_exit_signals]`::*::caller_errors` +(answered only at death), [`panic_safety`][panic_safety]. + +**R6. Deferred replies.** A handler may answer later: `PendingReply::defer()` +takes one of the instance's `MAX_OUTSTANDING` credits (or answers +`Overloaded` at once) and returns a storable `ReplyTo`. The token is answered +from a later turn, a job or `stopped()`; dropping it answers `ReplyDropped` +between turns, is committed at the turn's end inside a turn, and while the +owner exits is held until its death point. At the owner's death every open +token becomes `ActorStopped` with the final reason, wherever it is held; a +reply made once `NoProc` is observable for the owner is refused +(`Delivery::OwnerExited`) and the caller gets that reason. Queued requests +and deferred tokens are answered at the death point before any queued +payload or state destructor runs, also when the runtime drops the actor +(G1). The owner's deadline heap expires +deferred tokens (returning their credits) even when no caller polls. Callers +and protocols cannot tell a deferred reply from an immediate one. +Checked by: `reply_slot_loom` (every concurrent pair of the slot word), +[`deferred_replies`][deferred_replies], `reply::registry::tests`, +`death_boundary` (blocking destructors; both modes; runtime drop), +`failpoints::no_reply_succeeds_after_noproc_is_observable`. + +**R7. Pipe-to-self jobs.** `ctx.pipe*` runs work owned by the instance and +delivers `done(outcome)` as one message in a later turn; a panic arrives as +`JobError::Panicked`, a job deadline as `JobError::DeadlineExceeded` while the +work keeps its credit. Messages of cancelled or replaced (keyed) jobs are +fenced at dispatch; a job's emits stay FIFO with the terminal message last. +Owner exit cancels every job; they are joined after death, bounded by the +cleanup deadline, and reported as `unfinished_at_stop`. +`JobCtx::emit_at(at, ..)` with `at` past `JobOptions::deadline` is accepted: +the hard deadline takes the terminal notification when it passes and fences +everything after it, so the emit is dropped at its instant (its credit +returns then). It is not an error at the call: whether the deadline wins is +decided when it passes (the job may finish first, delivering the emit and +then its result), and a job checks one rule, the deadline, rather than two. +An `at` already reached is an immediate emit. +Checked by: [`deferred_jobs`][deferred_jobs]. ## 4. Lifecycle @@ -164,54 +245,73 @@ Checked by: [`startup_admission`][startup_admission], (both group kinds). (`ready_gate_loom` checks the benchmark scheduler's gate, not this barrier.) **L3. Stop is cooperative.** `ctx.stop()`, `ctx.stop_with_reason(r)`, -`ChildHandle::stop()`, `stop_with_reason(r)` and `shutdown()` request a stop: -the running handler finishes, `stopped()` runs, then the actor exits. Nothing -preempts a running handler or blocking code. The first committed reason wins; a -later stop or linked failure does not relabel it, but an actual panic in -cleanup still makes the exit abnormal. -Checked by: [`context_exit`][context_exit] (both modes), -`src/link_regression_tests.rs::late_link_failure_cannot_relabel_an_already_requested_normal_stop`. - -**L4. Exit reasons.** Every actor publishes exactly one `ExitReason`: +`ChildHandle::stop()` and `stop_with_reason(r)` are self-exits +or administrative stops: they ignore `trap_exit`, the running handler +finishes, `stopped()` runs, then the actor exits. Nothing preempts a running +handler or blocking code, a kill included. The first stop's reason wins over +later stops; until the death point D a kill makes it `Killed`, and a panic or +an untrapped exit signal or link notice replaces it unless a kill was received +(OTP: a crash or signal during `terminate` replaces the reason). After D the +reason is final. +Checked by: `Lifecycle!KilledFromKill`, `Lifecycle!LateSignalWins` (TLC); +[`context_exit`][context_exit] (both modes), `src/link.rs::tests::reason_precedence_until_death`, +`src/link_regression_tests.rs::late_link_failure_relabels_an_already_requested_normal_stop`, +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{late_signal_replaces_the_requested_reason, kill_during_stopped_supersedes_and_panic_does_not_replace_it}`; +[`exit_signals_sim`][exit_signals_sim]`::{kill_races_stopped, late_signal_races_the_death_point}`. + +**L4. Exit reasons.** Every actor publishes exactly one `ExitReason` +(`#[non_exhaustive]`): | Reason | Arises from | |---|---| -| `Normal` | `ctx.stop()`, `ChildHandle::stop()` | -| `Shutdown` | `ChildHandle::shutdown()`, `stop_with_reason(Shutdown)`, supervisor or system shutdown, a linked peer's `Shutdown` (non-trapping) | -| `Panic(msg)` | a panic in `started()` (`"panic in started(): .."`), a handler, `exit_received` or `Down` handler (`"panic in handler: .."`), `stopped()` (`"panic in stopped(): .."`, only if not already abnormal); a linked peer's `Panic`; a destructor panic on an otherwise `Normal` exit (`"actor state destructor: .."`, `"queued message destructor: .."`) | -| `Kill` | `stop_with_reason(Kill)`, a linked peer's `Kill`; tasks only: the actor future dropped without completing (runtime shutdown, abort, never polled), unless a non-`Normal` cause was already committed | - -`is_abnormal()` is true for `Panic` and `Kill`. +| `Normal` | `ctx.stop()`, `ChildHandle::stop()`, a parent's `Normal` exit to a trapping child | +| `Shutdown` | `stop_with_reason(Shutdown)`, supervisor or system shutdown, an exit signal or link notice with `Shutdown` (non-trapping) | +| `Panic(msg)` | a panic in `started()` (`"panic in started(): .."`), a handler, `exit_received` or `Down` handler (`"panic in handler: .."`), `stopped()` (`"panic in stopped(): .."`); an exit signal or link notice with `Panic` | +| `Kill` | `stop_with_reason(Kill)` (a stop, not a kill), a `Kill` link notice (non-trapping) | +| `Killed` | `ChildHandle::kill()`, `exit(Kill)`, a supervisor's or system's kill escalation, a `Killed` link notice; the runtime dropping the actor unless a non-`Normal` reason was requested | +| `NoProc` | a `link` to an actor past its death point (non-trapping caller), a `NoProc` signal or notice | + +`is_abnormal()` is true for `Panic`, `Kill`, `Killed` and `NoProc`. +Destructor panics after the death point do not change the reason (L6). Checked by: [`exit_signal_matrix`][exit_signal_matrix], [`context_exit`][context_exit], -[`panic_safety`][panic_safety], `src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, -`src/actor_cleanup.rs::destructor_failure_only_escalates_normal_even_without_a_committed_cause`, -[`exit_reason`][ex_exit_reason] and [`native_lifecycle`][ex_native_lifecycle] examples. +[`otp_exit_signals`][otp_exit_signals], [`panic_safety`][panic_safety], +`src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, +[`exit_reason`][ex_exit_reason], [`native_lifecycle`][ex_native_lifecycle] and +[`exit_signals`][ex_exit_signals] examples. **L5. Panics in hooks.** A panic in `started()` stops the actor without calling `stopped()` and without handling any message. A panic in a handler loses that message, then `stopped()` runs. A panic in `stopped()` is caught; `join()` -returns and the reason becomes `Panic` unless it was already abnormal. +returns and the reason becomes `Panic`, replacing an earlier requested reason +unless a kill was received (then `Killed` stands). `stopped()` does not run +after a kill, an untrapped exit signal or link notice, or a runtime drop +(see X3, X4); one that already began finishes. Checked by: `src/{tasks,threads}/actor.rs::panic_in_started_stops_actor`, `panic_in_handler_stops_actor`, `panic_in_stopped_still_completes`, `panic_during_early_stop_is_reported`; [`panic_safety`][panic_safety], [`foreign_panics::panicking_subscriber_keeps_handler_reason_and_runs_stopped`][foreign_panics], [`semantics_gaps`][semantics_gaps]`::tasks_contract::panics_in_hooks_follow_the_contract_on_every_placement` -(reason prefix, handled count and `stopped()` calls). - -**L6. Destructor panics are secondary.** A panicking `Drop` of the actor state -or of a queued message turns only an otherwise `Normal` exit into `Panic`; a -committed `Shutdown`, `Kill` or panic keeps its reason and the queued failure -is kept in `ChildHandle::mailbox_cleanup_failure()`. A panic payload whose own -destructor panics is forgotten (`dispose_panic`) to stop the cascade. +(reason prefix, handled count and `stopped()` calls); +`Lifecycle!SkipHook` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{kill_during_stopped_supersedes_and_panic_does_not_replace_it, stopped_skipped_log::skipped_stopped_is_logged_at_debug}`; +[`exit_signals_sim::kill_races_stopped`][exit_signals_sim]; lifecycle traces +`kill_during_stopped`, `untrapped_signal_skips_stopped` ([`lifecycle_trace`][lifecycle_trace]). + +**L6. Destructor panics are secondary.** The actor state and queued +messages are dropped after the death point, so a panicking `Drop` does not +change the exit reason; it is recorded in `ChildHandle::cleanup_failure()` +(`mailbox_cleanup_failure()` is a deprecated alias), and a Transient child +restarts on it. A panic payload whose own destructor panics is forgotten +(`dispose_panic`) to stop the cascade. Checked by: [`mailbox_drop_panic`][mailbox_drop_panic] (5 placements x 1/2/256 destructors), [`panic_safety`][panic_safety], [`native_mailbox_drop`][native_mailbox_drop], -`queued_drop_panic`/`queued_drop_preserves_shutdown` lifecycle traces. +`queued_drop_panic`/`queued_drop_shutdown_restarts` lifecycle traces. **L7. Exit publication is last.** The exit reason is published after `stopped()`, after queued messages and reply senders are released, after links have been signalled and outgoing monitors cancelled. `join()`, `wait_exit()` -and `ChildHandle` waits return only then; afterwards `send`/`request` return -`ActorStopped`. +and `ChildHandle` waits return only then; `send`/`request` return `NoProc` +from the death point on, which comes before publication. Checked by: [`shutdown_publication`][shutdown_publication], `src/link_regression_tests.rs::never_polled_completion_waits_for_actor_payload_destruction`, [`actor_scenarios`][actor_scenarios] (post-exit requests), @@ -222,60 +322,121 @@ outgoing monitors cancelled before publication. ## 5. Links, monitors and trapped exits **X1. Down exactly once.** `ctx.monitor(&handle)` (requires `Handler`) -yields exactly one `Down { monitor_ref, reason }` per monitor that is not -cancelled, carrying the target's published reason. An already-exited target -produces `Down` immediately. Each call returns a distinct `MonitorRef`; the +yields exactly one `Down { monitor_ref, actor, reason }` per monitor that is not +cancelled: the target's final reason if registered before its death point D, +`NoProc` (at once) if after it. Each call returns a distinct `MonitorRef`; the target is unaffected. -Checked by: [`actor_scenarios::demonitor_before_exit_suppresses_notification_and_races_do_not_duplicate`][actor_scenarios], +Checked by: `Lifecycle!DownAtMostOnce`, `Lifecycle!NoProcIffLate`, `Lifecycle!DownDelivered` (TLC); +[`link_fence_loom`][link_fence_loom]`::monitor_racing_death_gets_exactly_one_down`; +[`exit_signals_sim::link_and_monitor_race_the_death_point`][exit_signals_sim]; +[`actor_scenarios::demonitor_before_exit_suppresses_notification_and_races_do_not_duplicate`][actor_scenarios], [`exit_signal_matrix`][exit_signal_matrix], `src/*/actor.rs::monitor_*`, `multiple_monitors_each_get_own_ref_and_down`, +[`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals], [`cancellation_safety::monitor_and_link_setup_requests_are_cancel_safe`][cancellation_safety]. -**X2. Demonitor and late registration.** `demonitor` before the target exits -suppresses `Down`; a `Down` already queued may still arrive. A monitor +**X2. Demonitor and late registration.** `demonitor` flushes, as OTP's +`demonitor(Ref, [flush])`: once it returns, in both modes, no `Down` of that +monitor is enqueued or handled, including one already in the mailbox (the +late `NoProc` included); a `Down` whose dispatch began before it returned +was already received and is handled. A `Down` that user code sends itself is +an ordinary message. A monitor registered by an actor that is already stopping is inert (no waiter, no `Down`). -Checked by: `src/*/actor.rs::demonitor_before_target_dies_suppresses_down`, +Likewise `ctx.link`/`ctx.try_link` called through a `Context` whose actor's +exit already began, or which is past its death point (for example from a +helper that outlives the actor), is inert: no record, no notice, no `NoProc`, +and `try_link` returns `Ok(())` (a dead process cannot call `link` in OTP; +design otp-exit-signals.md, "Sender lifetime"). +Checked by: `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`; the +`NoDemonitorFlush` control runs a flushed `Down`); +`src/*/actor.rs::{demonitor_before_target_dies_suppresses_down, demonitor_flushes_a_queued_down}`, +`src/monitor.rs::tests::demonitor_flushes_a_queued_down`; +[`link_fence_loom`][link_fence_loom]`::demonitor_flushes_a_down_in_dispatch` +(a dispatch racing `demonitor` on another thread); +[`failpoints`][failpoints]`::{demonitor_flushes_a_down_in_dispatch, native::demonitor_flushes_a_down_in_dispatch}` +(site `monitor.down_before_dispatch`: `demonitor` returns while the `Down` +is in dispatch; the window is cross-thread, so a failpoint, not the +simulator, reaches it); +`link::tests::link_from_an_exiting_or_dead_caller_is_inert`, +[`otp_exit_signals::tasks_mode::link_from_a_dead_actor_is_inert`][otp_exit_signals], `closed_monitor_table_rejects_late_registration`, [`leaks::monitor_waiters_released_on_demonitor_owner_exit_and_target_exit`][leaks]. -**X3. Link propagation.** When a linked actor exits: +**X3. Exit signals and link propagation.** An exit signal +(`ChildHandle::exit`, `ctx.exit`, `kill`) or a linked actor's death notice is +applied at the receiver by OTP's `exit_signal/2` rules, with the trap flag +read when it arrives: -| Peer exits with | Non-trapping peer | Trapping peer (`ctx.trap_exit(true)`) | +| Signal or notice | Non-trapping receiver | Trapping receiver (`ctx.trap_exit(true)`) | |---|---|---| +| `Kill` signal (`kill()`, `exit(Kill)`) | ends `Killed` | ends `Killed` | | `Normal` | nothing | `exit_received(Exit { from, reason: Normal })` | -| `Shutdown`, `Panic`, `Kill` | stopped cooperatively, exits with the same reason (unless it already committed one); propagates further | `exit_received(Exit { from, reason })` | - -Propagation is transitive through non-trapping actors. Links work across modes. -Checked by: `src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, +| any other reason (a `Kill` notice included) | ends with that reason after its current handler, without `stopped()`, queue discarded; propagates further | `exit_received(Exit { from, reason })` | +| from the receiver's parent (`start_linked` caller, supervisor, system) | as above | a parent exit is queued: when dispatched the actor exits with that reason through `stopped()`; `exit_received` never sees it | + +Propagation is transitive through non-trapping actors. Links work across +modes. Signals arriving during `started()` take effect when it returns, with +the trap flag read then. `ctx.exit` works until the sender's death point, +also from a cloned context on another thread: the death point waits for a +signal in flight, so a signal sent before it reaches its target before the +sender's death notice (`Lifecycle!ExitAfterMessages`); one sent after it is +inert. +Checked by: `Lifecycle!TrapRespected`, `Lifecycle!NormalSignalIgnored`, `Lifecycle!KilledFromKill`, +`Lifecycle!SkipHook`, `Lifecycle!LinkedPeersStop`, `Lifecycle!ShutdownEndsChild` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{exit_signals_follow_exit_signal_2, kill_is_untrappable_and_propagates_killed, ctx_exit_sends_from_the_calling_actor, parent_exit_ends_a_trapping_child_normally, signals_during_started_are_resolved_with_the_flag_set_in_started}`, +`threads_mode::kill_exit_and_noproc_in_threads_mode`; +`failpoints::an_exit_signal_is_not_overtaken_by_its_senders_death_notice`; +`src/link.rs::tests::signal_rules_follow_exit_signal_2`; +`src/tasks/actor.rs::link_propagates_panic_to_non_trapping_peer`, `link_delivers_exit_to_trapping_peer`, `link_normal_exit_not_propagated_to_non_trapping_peer`, `chain_propagation_through_non_trapping_middle` (and threads equivalents), [`exit_signal_matrix`][exit_signal_matrix], [`linked_exit_macro`][linked_exit_macro], -`src/link_regression_tests.rs::linked_runtime_abort_cancels_an_idle_threads_actor` (cross-mode). - -**X4. Direct stops are untrappable, propagated Kill is trappable.** Every -direct stop (`stop`, `shutdown`, `stop_with_reason(any)`) ignores `trap_exit`. -A `Kill` arriving over a link is trappable, so one kill cannot take down a -graph of trapping observers. This follows Erlang's `kill`/`killed` split, except the -propagated value is still `Kill` (section 11). -Checked by: [`kill_links`][kill_links] (both modes). - -**X5. Link registration.** Linking to an already-exited peer delivers that -peer's actual reason once, under X3, and retains no link (Erlang reports -`noproc` instead). Repeating a live link and linking to oneself are no-ops. -`unlink` stops future propagation but does not retract a signal in flight. -`start_linked` registers the link before the child's `started()`. A link -installed while the peer is between draining its links and publishing its exit -still receives the exit. See section 11 for the planned `NoProc`. -Checked by: `src/tasks/actor.rs::link_to_already_dead_actor_delivers_signal`, -`link_to_already_dead_delivers_exactly_once_to_trapping_peer`, `duplicate_link_is_idempotent`, -`unlink_prevents_signal_delivery`; `src/link_regression_tests.rs::linking_in_post_drain_pre_completion_window_delivers_exit`, -`start_linked_registers_before_async_started`; `src/native_link_regression_tests.rs::start_linked_registers_before_threads_started`. - -**X6. Runtime teardown still notifies.** A tasks actor whose future is dropped -by its runtime, polled or never polled, still signals its links and monitors -with `Kill`, and the notice is trappable. -Checked by: `src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, +`src/link_regression_tests.rs::linked_runtime_abort_cancels_an_idle_threads_actor` (cross-mode); +lifecycle traces ([`lifecycle_trace`][lifecycle_trace]); [`exit_signals`][ex_exit_signals] example. + +**X4. Kills are untrappable; administrative stops are not signals.** Only a +`Kill` *signal* is untrappable and produces `Killed`. A `Kill` or `Killed` +arriving as a link notice is trappable, so one kill cannot take down a graph +of trapping observers (OTP's `kill`/`killed` split). `stop`, `shutdown` and +`stop_with_reason(any)` are administrative stops (as `gen_server:stop`): +untrappable, `stopped()` runs, and `stop_with_reason(Kill)` ends `Kill`. +Checked by: [`kill_links`][kill_links] (both modes), +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{kill_is_untrappable_and_propagates_killed, administrative_kill_reason_runs_stopped_and_ends_kill}`; +`Lifecycle!KilledFromKill` (TLC). + +**X5. Link registration, NoProc and fences.** A link records both ends in +one transaction. Linking to an actor past its death point D keeps no +relationship and delivers one `NoProc` notice: a trapping caller gets +`Exit { reason: NoProc }`, a non-trapping caller ends `NoProc`; +`ctx.try_link` returns `Err(NoProc)` and delivers nothing. A link racing D +gets exactly one notice, the real reason or `NoProc`. Repeating a link +(including one whose notice is pending) and linking to oneself are no-ops. +`unlink` and `demonitor` are fences: once they return, no notice of that edge +takes effect; an `Exit` already queued stays, a `Down` already queued is +flushed (X2). Each edge has a generation, so a stale +notice never applies to a later link. `start_linked` registers the link +before the child's `started()`. +Checked by: `Lifecycle!NoProcIffLate`, `Lifecycle!NoEdgeToDead`, `Lifecycle!CancelFenced` (TLC, `MCLifecycleFence`); +[`link_fence_loom`][link_fence_loom] (the production link table and monitor +fence under Loom: `unlink_fences_a_death_notice`, `link_racing_death_gets_exactly_one_notice`, +`relink_after_unlink_ignores_the_stale_generation`, `death_racing_unlink_and_a_new_link`, +`mutual_death_delivers_at_most_one_notice`, `concurrent_links_install_one_edge`, +`demonitor_fences_the_down`, `observer_close_fences_the_down`); +[`exit_signals_sim`][exit_signals_sim]`::{link_and_monitor_race_the_death_point, unlink_and_demonitor_fence_delivery}`; +`src/link.rs::tests::{closure_is_terminal_and_delivers_each_edge_once, try_link_to_a_dead_actor_keeps_no_record, unlink_fences_a_notice_taken_at_death}`; +[`otp_exit_signals::tasks_mode::registrations_after_death_report_noproc`][otp_exit_signals]; +`src/tasks/actor.rs::duplicate_link_is_idempotent`, `unlink_prevents_signal_delivery`; +`src/native_link_regression_tests.rs::start_linked_registers_before_threads_started`; +lifecycle traces `late_registrations_get_noproc`, `unlink_fences_a_notice_taken_at_death`, +`demonitor_fences_a_down`. + +**X6. Runtime teardown still notifies.** An actor whose future or thread is +dropped by its runtime, polled or never polled, ends `Killed` in both modes +(unless a non-`Normal` reason was requested), without `stopped()`, and still +signals its links (trappable) and monitors. +Checked by: `Lifecycle!RuntimeDropReason` (TLC); +`src/link_regression_tests.rs::never_polled_actor_propagates_runtime_shutdown_to_peer`, `polled_actor_propagates_runtime_shutdown_to_peer`, `never_polled_actor_notifies_monitor_on_runtime_shutdown`, -`linked_runtime_abort_is_trappable`. +`linked_runtime_abort_is_trappable`; lifecycle trace `runtime_drop_ends_killed`. **X7. Ordering.** Trapped `Exit` and `Down` are ordinary mailbox messages: they queue behind messages already in the mailbox and never interrupt a running @@ -293,7 +454,7 @@ source's sends, noise at the observer, PCT and bounded DFS), the fuzz X7 oracle, **V1. Restart table.** While the supervisor is open: -| Mode | `Normal` / `Shutdown` exit | `Panic` / `Kill` exit | +| Mode | `Normal` / `Shutdown` exit | `Panic` / `Kill` / `Killed` / `NoProc` exit, or a cleanup failure | |---|---|---| | Permanent (default) | restart | restart | | Transient | stay stopped | restart | @@ -324,30 +485,45 @@ Checked by: `Cohorts!StrategySelection`, `Cohorts!WholeCohortBarrier`, **V3. At most one live generation.** A slot never has two live generations: a replacement starts only after the previous actor, its registered helpers and -its owned cleanup actually finished. If cleanup misses its `shutdown_timeout`, -the slot is stuck, the supervisor fails and nothing replaces it. +its owned cleanup actually finished. When the supervisor stops a child +(shutdown, a OneForAll/RestForOne round, a startup rollback), a child that has +not reached its death point `shutdown_timeout` after its signal is killed +(`Killed`, design E3); if it misses a second `shutdown_timeout` the slot is +stuck. A child that exits on its own is stuck if its cleanup misses +`shutdown_timeout` from its exit (no kill: it would relabel its reason). A +stuck slot fails the supervisor and nothing replaces it. Checked by: `Supervision!AtMostOneGeneration`, `Cohorts!AtMostOneGeneration`, `NestedSupervision!AtMostOneGeneration`; [`supervision::cleanup_deadline_escalates_without_overlapping_a_live_generation`][supervision], -`repeated_crash_producer_and_shutdown_races_never_overlap_generations`. +`repeated_crash_producer_and_shutdown_races_never_overlap_generations`, +`stuck_stopped_hook_triggers_cleanup_deadline_before_terminal_publication`; +[`otp_exit_signals::tasks_mode::supervisor_shutdown_is_the_parents_exit_signal`][otp_exit_signals] +(the kill at `shutdown_timeout`). **V4. Generations and publication.** Each construction attempt increments the slot's generation. `SupervisedChild::current()` returns only the latest live, ready generation; `wait_for_generation(n)` waits for a ready generation newer than `n`. Old `ActorRef`s stay bound to the old instance and fail with -`ActorStopped`; queued messages are discarded, never replayed. +`NoProc` (queued requests got `ActorStopped` with its final reason); queued +messages are discarded, never replayed. Checked by: `Supervision!PublishedIsLive`, `Cohorts!PublishedIsLive`; [`supervisor_policies`][supervisor_policies] (request fences), [`semantics_gaps`][semantics_gaps]`::supervision_contract::queued_messages_are_not_replayed_into_a_replacement`, [`supervision_recovery`][ex_supervision_recovery] example. -**V5. Restart budget.** At most `max_restarts` retry grants per rolling -`restart_window` (default 5 per 30 s), shared by the whole supervisor. Initial -construction is free; every later factory call counts, including failed -factories and collateral OneForAll/RestForOne children (unlike OTP, which counts -rounds). Capacity is reserved before backoff, refunded if the backoff is -cancelled or superseded, and stamped as a grant at admission. An exhausted -window or `max_restarts = 0` fails at once, without waiting for backoff. -Exhaustion fails the supervisor and stops its children in reverse order. +**V5. Restart budget.** At most `max_restarts` restart actions per rolling +`restart_window` (default 5 per 30 s), shared by the whole supervisor, as OTP's +intensity and period. Initial construction is free. A OneForOne child retry +(and any retry before the tree is ready) is one action: its capacity is +reserved before backoff, refunded if the backoff is cancelled or superseded, +and stamped as a grant at admission, so a failed factory or `started()` after +admission still counts. A OneForAll/RestForOne recovery round is one action, +however many children it rebuilds: the coordinator takes its grant when it +selects the round, before stopping the cohort (OTP's `add_restart`), and the +round's rebuilds take none. A failed replacement inside a round requests a new +round, charged again (`try_again_restart`); a superseded round keeps its +charge. An exhausted window or `max_restarts = 0` fails at once, without +waiting for backoff. Exhaustion fails the supervisor and stops its children in +reverse order. Checked by: `Supervision!BudgetBounded`; Kani `restart_window_admits_exactly_within_budget`; [`supervision::restart_budget_is_shared_between_siblings_and_exhaustion_closes_tree`][supervision], `disabled_retries_fail_without_waiting_for_a_long_backoff`, @@ -366,16 +542,22 @@ Checked by: Kani `backoff_bound_is_capped`, `backoff_jitter_stays_in_range`, [`semantics_gaps`][semantics_gaps]`::supervision_contract::collateral_children_add_no_backoff`. **V7. Startup.** Children start serially in declaration order. A tree reports -ready only when every child is ready; a non-restartable failure (or exhausted -retries) rolls back the started children in reverse order. Each attempt has a -`startup_timeout` (default 5 s) that counts as a failure. A child that stops -`Normal` (or `Shutdown`) before its initial readiness is a startup failure and, -if Transient, fails the start, as a failed child `init` fails an OTP -supervisor's start. A replacement that stops before readiness is a failed -restart attempt and is retried (V1). Factories run on the child's -group; a factory error or panic is a failed attempt. Cancelling the start future -before readiness rolls the tree back. -Checked by: `Supervision!ReadyMeansAllStarted`; [`supervision::failed_partial_startup_rolls_back_without_starting_later_children`][supervision], +ready only when every child is ready. A child's first start that fails for +any reason (factory error or panic, panic or stop in `started()`, a missed +`startup_timeout`, default 5 s, placement or group closure) fails the start +whatever the restart type, with no retry, as OTP's `start_children` returns +`failed_to_start_child`: the started children roll back in reverse order and +`start()` returns `SupervisorError::FailedToStartChild` only after the +rollback (every child terminal, or `Failed` once its cleanup is stuck). A +ready child's exit while later siblings start is recorded and processed under +the strategy once the tree commits ready, as OTP handles the queued `'EXIT'` +after `init`; if the start aborts, it is part of the rollback. A replacement +that fails before readiness is a failed restart attempt and is retried (V1). +Factories run on the child's group. Cancelling the start future before +readiness rolls the tree back. +Checked by: `Supervision!ReadyMeansAllStarted`, `Supervision!RestartFollowsPolicy` (no restart before the tree is ready), `Cohorts!FirstStartOnce`; [`supervision::failed_partial_startup_rolls_back_without_starting_later_children`][supervision], +`otp_exit_signals::first_start_failure_aborts_start_whatever_the_restart_type`, `ready_child_exit_during_startup_is_processed_afterwards`, `start_returns_its_error_after_rollback`, +[`exit_signals_sim::ready_child_exit_races_the_tree_commit`][exit_signals_sim], `transient_startup_timeout_retries_after_actual_cleanup`, `normal_pre_ready_exit_records_real_cause_and_fails_transient_startup`, `transient_replacement_exiting_before_readiness_is_retried`, [`supervision_trace::trace_replacement_exiting_before_readiness_is_retried`][supervision_trace], `failed_factories_cancel_owned_helpers_before_retry_and_catch_panics`, @@ -392,9 +574,15 @@ Checked by: `NestedSupervision!NoEarlyParentJoin`, `NestedSupervision!DoneMeansD [`nested_supervision`][nested_supervision], [`tree_trace`][tree_trace]. **V9. Supervisor shutdown.** Closing a supervisor never reopens it and starts -or publishes nothing afterwards. Ready children get their stop turn in reverse -declaration order, each bounded by its `shutdown_timeout` (default 5 s); at the -report deadline the remaining siblings are signalled. "Done" means no live +or publishes nothing afterwards. Ready children get their turn in reverse +declaration order. Each receives its parent's `Shutdown` exit signal (E3): a +non-trapping child ends `Shutdown` without `stopped()`; a trapping child +handles the messages queued before it (`exit_pending()` is `true` there), then +runs `stopped()` and ends `Shutdown`, never seeing it in `exit_received`. Each +turn is bounded by the child's `shutdown_timeout` (default 5 s), then the +child is killed (V3); at the report deadline the remaining siblings are +signalled. `ActorSystem::shutdown(grace)` signals standalone actors the same +way and kills those left at `grace`. "Done" means no live generation and every controller finished. Supervisors observe children through internal completion state, not their mailboxes, so a message backlog does not delay observation. @@ -403,7 +591,10 @@ Checked by: `Supervision!NoStartAfterClose`, `Supervision!ReverseShutdown`, `NestedSupervision!NoStartAfterClose`, `Supervision!ClosedLeadsToDone` (TLC-only, assumes cleanup ends); [`supervision::shutdown_during_backoff_prevents_new_factory_invocation`][supervision], `supervisor_report_deadline_signals_remaining_siblings`, [`failpoints`][failpoints], -[`semantics_gaps`][semantics_gaps]`::supervision_contract::message_backlog_does_not_delay_restart`. +[`semantics_gaps`][semantics_gaps]`::supervision_contract::message_backlog_does_not_delay_restart`; +`Lifecycle!ShutdownEndsChild` (TLC); +[`otp_exit_signals`][otp_exit_signals]`::tasks_mode::{supervisor_shutdown_is_the_parents_exit_signal, exit_pending::queued_work_sees_the_supervisors_pending_shutdown}`; +[`exit_signals_sim::supervisor_shutdown_races_a_trapping_childs_queue`][exit_signals_sim]. ## 7. Systems and execution groups @@ -497,7 +688,13 @@ is tasks-only). **T4. Clocks.** Delays use monotonic time (Tokio timers in tasks, so they follow paused Tokio time in tests; `recv_timeout` in threads). threads: each timer is -one OS thread. Nothing is persisted across process restarts. +one OS thread. Nothing is persisted across process restarts. Request +deadlines are instants on the `spawned_rt` clock: `request`, +`request_with_timeout` and `Response::from` wait on the runtime's timer (paused +Tokio time applies, and a shortened deadline re-arms it); token and job +deadlines, `emit_at` and a raw receiver awaited directly are serviced by the +owner's deadline heap on the real clock (the simulated clock under +`spawned_sim`), not by paused Tokio time. Checked by: the paused-time tests (`semantics_gaps::tasks::send_after_fires_once_no_earlier_than_its_delay`, `send_interval_is_fixed_delay_and_never_catches_up`, `default_request_timeout_is_five_seconds_from_first_poll`) and the simulator's virtual clock for tasks; `src/threads/timer_tests.rs::threads_send_after_uses_the_monotonic_clock_over_long_delays`, @@ -563,9 +760,9 @@ stopped after `Shutdown`), - **Erlang/OTP equivalence.** See section 11 for each current difference. - **Liveness under schedules nobody explored.** Tests run real schedulers; models abstract time and handler bodies (see [TESTING](../TESTING.md#how-spawned-is-verified)). - The base actor lifecycle (links, monitors, requests around shutdown) has no - TLA+ model of the v0.7 behaviour; `Lifecycle.tla` models the proposed v0.8 - contract ([next model](../../verification/tla/README.md#next-model)). + The base actor lifecycle (signals, links, monitors, requests around the + death point) is modelled by `Lifecycle.tla` (TLC only), and recorded + lifecycle traces are checked against it. ## 11. Differences from Erlang/OTP @@ -574,18 +771,25 @@ where Rust types add strictness, or where Spawned adds bounds Erlang lacks. Already the same as OTP: per-sender FIFO (D1), at-most-once delivery (D2), `gen_server:call`'s 5 s default timeout (R3), a stop dropping the mailbox like a -process exit (D4), linked exits propagating the same reason and `Normal` being -ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table (V1). +process exit (D4), exit signals and link notices following `exit_signal/2` +(X3), the `kill`/`killed` split (X4), `noproc` for late links, monitors and +calls and the unlink/demonitor fences (X1, X5, R1), a parent's exit ending a +trapping child through `stopped()` (X3, V9), `stopped()` skipped on kills and +untrapped signals (L5), and the Permanent/Transient/Temporary table (V1). The +remaining differences of the exit-signal design are listed in +[otp-exit-signals.md](otp-exit-signals.md#differences-from-otp-that-remain). | Today | OTP | Status | |---|---|---| -| Every direct stop (`stop`, `shutdown`, `stop_with_reason(any)`) ignores `trap_exit` (X4) | `exit(Pid, normal)` is ignored by a non-trapping process; every reason except `kill` is trappable | Planned v0.8, rank 5e: a `Normal` exit signal is ignored by a non-trapping actor, all reasons except `Kill` (including `Shutdown`) are trappable, only `Kill` is untrappable | -| Supervisor shutdown cancels a trapping child directly | `shutdown` reaches a trapping child as an `'EXIT'` message and it runs normal termination | Planned v0.8, rank 5e | -| A kill propagated over a link stays `Kill` (X4) | It arrives as `killed` | Planned v0.8, rank 5e: a distinct trappable `Killed` | -| Linking or monitoring an already-exited actor delivers its real exit reason (X1, X5) | `noproc` | Planned v0.8, rank 5e: `NoProc` | -| The restart budget counts every child factory call, collateral ones included (V5) | Intensity counts recovery rounds | Planned v0.8, rank 5f | -| Initial startup retries a Permanent or Transient child whose factory fails, panics or times out (V7), within the budget | `start_link` aborts on any child start error | Planned v0.8, rank 5f | -| An actor dropped by its runtime exits `Kill` in tasks; the threads fallback is `Panic("unexpected framework panic")` (L4) | No counterpart | Planned v0.8, rank 5g: one reason in both modes | +| v0.7 had no exit signals: every outside stop ignored `trap_exit` | `exit(Pid, normal)` is ignored by a non-trapping process; every reason except `kill` is trappable | Aligned in v0.8 (rank 5e, #240): `exit`/`kill` follow `exit_signal/2` (X3) | +| `stop`, `shutdown`, `stop_with_reason` return at once and stop after the current handler (X4) | `gen_server:stop` waits for the backlog | Bound: stop latency is one handler; they stay administrative stops, distinct from exit signals | +| v0.7 supervisor shutdown cancelled a trapping child directly | `shutdown` reaches a trapping child as an `'EXIT'` message and it runs normal termination | Aligned in v0.8 (rank 5e, #240): a parent exit (V9) | +| v0.7 propagated a kill over a link as `Kill` | It arrives as `killed` | Aligned in v0.8 (rank 5e, #240): `Killed` (X4) | +| v0.7 linking or monitoring an already-exited actor delivered its real exit reason | `noproc` | Aligned in v0.8 (rank 5e, #240): `NoProc` (X1, X5) | +| A kill or late signal does not interrupt a running `stopped()` (L3) | The process dies at once | Rust-forced: the reason still becomes `Killed` or the signal's reason | +| v0.7 charged the restart budget for every child factory call, collateral ones included | Intensity counts restart actions: one per `one_for_all`/`rest_for_one` round | Aligned in v0.8 (rank 5f, #240): a OneForAll or RestForOne recovery round takes one grant however many children it rebuilds (V5) | +| Initial startup retried a Permanent or Transient child whose factory failed, panicked or timed out (V7), within the budget | `start_link` aborts on any child start error | Aligned in v0.8 (rank 5f, #240): any first-start failure fails `start()` | +| v0.7 reported a runtime-dropped actor as `Kill` (tasks) or `Panic("unexpected framework panic")` (threads) | No counterpart | v0.8 (rank 5g, #240): `Killed` in both modes (X6) | | Stop is cooperative: a running handler or blocking call is never interrupted (L3) | `kill` ends a process at once | Rust-forced: Rust cannot preempt or kill a thread | | Panics are caught only with unwinding (P2) | Process crashes are always isolated | Rust-forced: `panic = "abort"` and a panicking panic hook end the process | | No per-actor memory accounting | `max_heap_size` | Rust-forced (no per-actor heap); opt-in size hook proposed, rank 18a | @@ -595,15 +799,11 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table ## 12. Planned changes to these semantics -- v0.8, rank 5e, exit signals as in OTP: trappable - `Shutdown`, `Normal` ignored by non-trapping actors, `Killed` for propagated - kills, `NoProc` for links and monitors to exited actors, normal termination - for a trapping child on supervisor shutdown (changes X1, X3, X4, X5, L3, L4). -- v0.8, rank 5f: the restart budget counts recovery rounds (changes V5). -- v0.8, rank 5g: one exit reason for runtime-dropped actors in both modes (changes L4, X6). -- Design for 5e, the 5f startup rule and 5g, accepted (not implemented): - [otp-exit-signals.md](otp-exit-signals.md). Reason-bearing `ActorStopped` - and `NoProc` for requests wait for rank 11 (#239). +- v0.8, ranks 5e, 5f (startup) and 5g, exit signals, startup and runtime + drop as in OTP ([otp-exit-signals.md](otp-exit-signals.md)): implemented on + the exit-signals branch (#240, on top of rank 11's #239) and described in + L3-L6, X1-X6, V1, V3, V5 (the restart budget counts recovery rounds), V7 + and V9 above. - Rank 11, deferred typed replies, forwarding and pipe-to-self: a reply may leave the handler's turn; adds exactly-one completion on panic and timeout. @@ -622,11 +822,17 @@ ignored by non-trapping peers (X3), and the Permanent/Transient/Temporary table See the [ranked backlog](../../ROADMAP.md#ranked-backlog). +[deferred_replies]: ../../concurrency/tests/deferred_replies.rs +[deferred_jobs]: ../../concurrency/tests/deferred_jobs.rs [actor_scenarios]: ../../concurrency/tests/actor_scenarios.rs [cancellation_safety]: ../../concurrency/tests/cancellation_safety.rs [cohort_trace]: ../../concurrency/tests/cohort_trace.rs [context_exit]: ../../concurrency/tests/context_exit.rs [cross_backend]: ../../concurrency/tests/cross_backend.rs +[otp_exit_signals]: ../../concurrency/tests/otp_exit_signals.rs +[exit_signals_sim]: ../../concurrency/tests/exit_signals_sim.rs +[link_fence_loom]: ../../concurrency/tests/link_fence_loom.rs +[lifecycle_trace]: ../../concurrency/tests/lifecycle_trace.rs [delivery_simulation]: ../../concurrency/tests/delivery_simulation.rs [execution_groups]: ../../concurrency/tests/execution_groups.rs [exit_signal_matrix]: ../../concurrency/tests/exit_signal_matrix.rs @@ -654,6 +860,7 @@ See the [ranked backlog](../../ROADMAP.md#ranked-backlog). [supervisor_policies]: ../../concurrency/tests/supervisor_policies.rs [tree_trace]: ../../concurrency/tests/tree_trace.rs [ex_exit_reason]: ../../examples/exit_reason +[ex_exit_signals]: ../../examples/capabilities/src/bin/exit_signals.rs [ex_native_lifecycle]: ../../examples/capabilities/src/bin/native_lifecycle.rs [ex_panic_abort]: ../../concurrency/examples/panic_abort.rs [ex_protocol_timeouts]: ../../examples/capabilities/src/bin/protocol_timeouts.rs diff --git a/docs/research/supervision-models.md b/docs/research/supervision-models.md index 126c34c9b..045c68a4e 100644 --- a/docs/research/supervision-models.md +++ b/docs/research/supervision-models.md @@ -125,9 +125,10 @@ Read from `concurrency/src/supervision.rs`, `supervision/nested.rs` and the abnormal exit), `Temporary`: OTP's three types. Supervisor or system shutdown suppresses every mode. Stopping a Permanent child individually restarts it. - **Intensity:** `max_restarts` 5 per rolling `restart_window` 30 s, **shared by - the supervisor**, as in OTP. Initial construction is free; every later factory - call counts, including collateral reconstruction in a cohort and failed - factories. Exhaustion fails the whole supervisor and stops its children; + the supervisor**, as in OTP. Initial construction is free; a OneForOne retry + (including a failed factory) counts once, and since 0.8 a OneForAll or + RestForOne recovery round counts once however many children it rebuilds + (0.7 charged each collateral reconstruction). Exhaustion fails the whole supervisor and stops its children; `max_restarts = 0` fails at once. A nested subtree's failure "escalates through the adapter to its parent's restart policy" (`SupervisorSpec`). - **Backoff:** per child, capped exponential (`initial` 10 ms, `maximum` 5 s) diff --git a/examples/README.md b/examples/README.md index fc19d6728..5cf2e8d8b 100644 --- a/examples/README.md +++ b/examples/README.md @@ -25,12 +25,14 @@ Usage of each feature is documented in the [guide](../docs/GUIDE.md). | Dedicated-thread actor with async helpers | [dedicated_thread](capabilities/src/bin/dedicated_thread.rs) | n/a | | Global registry (`registry::register`/`whereis`) | [service_discovery](service_discovery), [registrations and fakes](capabilities/src/bin/registrations_and_fakes.rs) | [registrations and fakes](capabilities/src/bin/registrations_and_fakes.rs), [thread_only](thread_only/README.md) | | Exit reasons, `join`, `wait_exit`, monitors, links, trapped exits | [exit_reason](exit_reason) | [native_lifecycle](capabilities/src/bin/native_lifecycle.rs) | +| Exit signals as in OTP (`exit`/`kill`, trapping, `NoProc`, `try_link`, parent exits, `exit_pending`), v0.8 | [exit_signals](capabilities/src/bin/exit_signals.rs) | gap (threads mode is tested in `otp_exit_signals::threads_mode`) | | Execution groups (`ActorSystem`, `start_on`, owned helpers), v0.7 | [execution_groups](capabilities/src/bin/execution_groups.rs) | n/a | | Supervision, v0.7 | [supervision_recovery](capabilities/src/bin/supervision_recovery.rs) | n/a | | Fixed native actor pools / parked Linux pinned shards, v0.7 | [execution_profiles](capabilities/src/bin/execution_profiles.rs) | n/a | | Nested supervisors, OneForAll / RestForOne, v0.7 | [supervision_tree](capabilities/src/bin/supervision_tree.rs) | n/a | | Tokio-free dependency graph, protocols, lifecycle, monitors and links, v0.7 | n/a | [thread_only](thread_only/README.md) (isolated workspace) | | Actors on separate runtimes | [runtime_isolation](capabilities/src/bin/runtime_isolation.rs) | n/a | +| Deferred replies and pipe-to-self (rank 11) | [deferred_replies](capabilities/src/bin/deferred_replies.rs): delegation with `pipe_reply`, CPU offload with a job deadline, requests by id in `PendingReplies`, waiters instead of polling, owner stop | gap (native jobs are tested in `deferred_jobs`) | | Request timeout outcomes | [timeout_outcomes](capabilities/src/bin/timeout_outcomes.rs), [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | [protocol_timeouts](capabilities/src/bin/protocol_timeouts.rs) | | Draining a finite batch before stop | [graceful_drain](capabilities/src/bin/graceful_drain.rs) | gap | | Byte-budgeted pipeline (application adapter) | [byte_budget_pipeline](capabilities/src/bin/byte_budget_pipeline.rs) | gap | diff --git a/examples/capabilities/README.md b/examples/capabilities/README.md index e7fce4419..27196b320 100644 --- a/examples/capabilities/README.md +++ b/examples/capabilities/README.md @@ -25,6 +25,7 @@ tracking is opt-in; it adds per-turn clock/locking costs, not payload logging. | [streams_and_recipients](src/bin/streams_and_recipients.rs) | Share one message interface across actor types | Broadcast through typed recipients, reject stopped members, cancel an async source and release a blocking native iterator cooperatively. | | [registrations_and_fakes](src/bin/registrations_and_fakes.rs) | Test protocol clients and resolve named replacements | A fake implements the same protocol; managed names follow fresh generations; manual guards preserve newer registrations. | | [test_kit](src/bin/test_kit.rs) | Unit-test an actor without its real dependencies | A `spawned-test` mock answers a protocol and a probe records sends; a slow reply forces a timeout, a crashed dependency and a killed supervised child are injected, and the same mock runs in threads mode. | +| [exit_signals](src/bin/exit_signals.rs) | Stop, kill or observe actors with OTP's exit-signal rules | `exit(Normal)` ignored and `exit(Shutdown)` ending an actor without `stopped()`; trapped `Exit`; `kill()` ending a trapping actor `Killed`; late link/monitor giving `NoProc`, `try_link` returning `Err(NoProc)`; a supervisor's shutdown reaching a trapping child after its queued work, which sees `exit_pending()`. | | [native_lifecycle](src/bin/native_lifecycle.rs) | Handle failures from synchronous actors | Monitors, demonitoring, trapped exits and linked startup, with no ambient Tokio runtime. | | [coordinated_shutdown](src/bin/coordinated_shutdown.rs) | Migrate hand-wired client shutdown | Join producers, snapshot a live dependency, then join a two-group tree in reverse dependency order. | | [framework_metrics](src/bin/framework_metrics.rs) | Observe actor activity without modifying handlers | Configure sampled timing, issue requests and render process-wide Prometheus text. Metrics are explicitly enabled for this binary. | @@ -44,6 +45,7 @@ cargo run --locked -p spawned-capability-examples --bin streams_and_recipients cargo run --locked -p spawned-capability-examples --bin registrations_and_fakes cargo run --locked -p spawned-capability-examples --bin test_kit cargo run --locked -p spawned-capability-examples --bin native_lifecycle +cargo run --locked -p spawned-capability-examples --bin exit_signals cargo run --locked -p spawned-capability-examples --bin coordinated_shutdown cargo run --locked -p spawned-capability-examples --features metrics --bin framework_metrics ``` diff --git a/examples/capabilities/src/bin/byte_budget_pipeline.rs b/examples/capabilities/src/bin/byte_budget_pipeline.rs index 86447c6cc..d53c6cf2e 100644 --- a/examples/capabilities/src/bin/byte_budget_pipeline.rs +++ b/examples/capabilities/src/bin/byte_budget_pipeline.rs @@ -18,13 +18,21 @@ impl Message for Packet { struct Sink(usize); impl Actor for Sink {} impl Handler for Sink { - async fn handle(&mut self, packet: Packet, _: &Context) -> usize { - let Packet { bytes, credit } = packet; - let size = bytes.len(); - self.0 += size; - drop(bytes); - drop(credit); // Capacity returns after the payload is released. - size + async fn handle( + &mut self, + packet: Packet, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + let __reply_value: usize = { + let Packet { bytes, credit } = packet; + let size = bytes.len(); + self.0 += size; + drop(bytes); + drop(credit); // Capacity returns after the payload is released. + size + }; + reply.reply(__reply_value); } } struct Total; @@ -32,15 +40,25 @@ impl Message for Total { type Result = usize; } impl Handler for Sink { - async fn handle(&mut self, _: Total, _: &Context) -> usize { - self.0 + async fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.0); } } struct Forwarder(ActorRef); impl Actor for Forwarder {} impl Handler for Forwarder { - async fn handle(&mut self, packet: Packet, _: &Context) -> usize { - self.0.request(packet).await.unwrap() + async fn handle( + &mut self, + packet: Packet, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.0.request(packet).await.unwrap()); } } diff --git a/examples/capabilities/src/bin/coordinated_shutdown.rs b/examples/capabilities/src/bin/coordinated_shutdown.rs index bfc1bffe5..39369b487 100644 --- a/examples/capabilities/src/bin/coordinated_shutdown.rs +++ b/examples/capabilities/src/bin/coordinated_shutdown.rs @@ -31,18 +31,34 @@ struct Peers { stops: Stops, } impl Actor for Peers { + // The supervisor's shutdown is its exit signal (as in OTP): trapping exits + // makes it end through `stopped()`, where this cleanup runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.stops.lock().unwrap().push("peers"); } } impl Handler for Peers { - async fn handle(&mut self, msg: Peer, _: &Context) { + async fn handle( + &mut self, + msg: Peer, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.peers.push(msg.0); + reply.reply(()); } } impl Handler for Peers { - async fn handle(&mut self, _: Snapshot, _: &Context) -> Vec { - self.peers.clone() + async fn handle( + &mut self, + _: Snapshot, + reply: spawned_concurrency::PendingReply<'_, Vec>, + _: &Context, + ) { + reply.reply(self.peers.clone()); } } struct Importer { @@ -50,17 +66,35 @@ struct Importer { stops: Stops, } impl Actor for Importer { + // The supervisor's shutdown is its exit signal (as in OTP): trapping exits + // makes it end through `stopped()`, where this cleanup runs. + async fn started(&mut self, ctx: &Context) { + ctx.trap_exit(true); + } async fn stopped(&mut self, _: &Context) { self.stops.lock().unwrap().push("importer"); } } impl Handler for Importer { - async fn handle(&mut self, msg: Import, _: &Context) { + async fn handle( + &mut self, + msg: Import, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.peers.send(Peer(msg.0)).unwrap(); + reply.reply(()); } } impl Handler for Importer { - async fn handle(&mut self, _: Fence, _: &Context) {} + async fn handle( + &mut self, + _: Fence, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[tokio::main(flavor = "current_thread")] async fn main() { @@ -70,10 +104,7 @@ async fn main() { let stops: Stops = Arc::default(); // RestForOne rebuilds dependents retaining an earlier instance's reference. let mut tree = system - .supervisor(SupervisorConfig { - strategy: SupervisorStrategy::RestForOne, - ..SupervisorConfig::default() - }) + .supervisor(SupervisorConfig::default().strategy(SupervisorStrategy::RestForOne)) .unwrap(); let events = stops.clone(); let peers = tree diff --git a/examples/capabilities/src/bin/dedicated_thread.rs b/examples/capabilities/src/bin/dedicated_thread.rs index 1604e3aaa..80c9990aa 100644 --- a/examples/capabilities/src/bin/dedicated_thread.rs +++ b/examples/capabilities/src/bin/dedicated_thread.rs @@ -12,21 +12,29 @@ impl Message for Check { type Result = (usize, u64); } impl Handler for Worker { - async fn handle(&mut self, _: Check, _: &Context) -> (usize, u64) { - let root = thread::current().id(); - let workers = tokio::runtime::Handle::current().metrics().num_workers(); - let (tx, rx) = mpsc::channel(); - let helper = tokio::spawn(async move { - tokio::time::sleep(Duration::from_millis(1)).await; - assert_ne!(root, thread::current().id()); - tx.send(42).unwrap(); - }); - // This would stall a current-thread executor. The isolated runtime's - // one async worker can drive the helper independently of this root. - let answer = rx.recv_timeout(Duration::from_secs(5)).unwrap(); - helper.await.unwrap(); - assert_eq!(root, thread::current().id()); - (workers, answer) + async fn handle( + &mut self, + _: Check, + reply: spawned_concurrency::PendingReply<'_, (usize, u64)>, + _: &Context, + ) { + let __reply_value: (usize, u64) = { + let root = thread::current().id(); + let workers = tokio::runtime::Handle::current().metrics().num_workers(); + let (tx, rx) = mpsc::channel(); + let helper = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(1)).await; + assert_ne!(root, thread::current().id()); + tx.send(42).unwrap(); + }); + // This would stall a current-thread executor. The isolated runtime's + // one async worker can drive the helper independently of this root. + let answer = rx.recv_timeout(Duration::from_secs(5)).unwrap(); + helper.await.unwrap(); + assert_eq!(root, thread::current().id()); + (workers, answer) + }; + reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/deferred_replies.rs b/examples/capabilities/src/bin/deferred_replies.rs new file mode 100644 index 000000000..0134524a4 --- /dev/null +++ b/examples/capabilities/src/bin/deferred_replies.rs @@ -0,0 +1,460 @@ +//! Deferred replies and pipe-to-self (rank 11), asserted: +//! +//! 1. a router delegates a slow request with `pipe_reply` and keeps serving +//! its mailbox meanwhile; +//! 2. CPU work offloaded with a reserved `pipe_blocking` (Tokio's +//! blocking pool) and a job deadline; +//! 3. the ethrex `outgoing_request` pattern: request ids in a bounded +//! `PendingReplies`, answered when the "peer" responds, late responses +//! ignored; +//! 4. the `get_best_peer` waiter queue: callers wait (no polling) until +//! capacity frees, and `Overloaded` at the bound; +//! 5. owner stop with stored tokens: `stopped()` answers, death closes the +//! rest with `ActorStopped`. +use spawned_concurrency::message::Message; +use spawned_concurrency::tasks::{ + Actor, ActorRef, ActorStart, Context, Deadline, Handler, InsertErrorKind, JobError, JobOptions, + JobOutcome, PendingReplies, PendingReply, ReplyTo, +}; +use spawned_concurrency::{actor, protocol, ActorError, Response}; +use std::collections::VecDeque; +use std::time::Duration; + +// --------------------------------------------------------------------------- +// 1. Router delegating with pipe_reply +// --------------------------------------------------------------------------- + +#[protocol] +pub trait BackendProtocol: Send + Sync { + fn lookup(&self, millis: u64) -> Response; +} + +pub struct Backend; +#[actor(protocol = BackendProtocol)] +impl Backend { + #[request_handler] + async fn lookup(&mut self, msg: backend_protocol::Lookup, _: &Context) -> u64 { + spawned_rt::tasks::sleep(Duration::from_millis(msg.millis)).await; + msg.millis + } +} + +#[protocol] +pub trait RouterProtocol: Send + Sync { + fn route(&self, millis: u64) -> Response; + fn ping(&self) -> Response<&'static str>; +} + +pub struct Router { + backend: ActorRef, +} +struct Delegated { + res: JobOutcome>, + token: ReplyTo, +} +impl Message for Delegated { + type Result = (); +} + +#[actor(protocol = RouterProtocol)] +impl Router { + #[deferred_handler] + async fn route( + &mut self, + msg: router_protocol::Route, + reply: PendingReply<'_, u64>, + ctx: &Context, + ) { + let Ok(token) = reply.defer() else { return }; + // The downstream request inherits the caller's deadline. + let deadline = token.deadline(); + let downstream = self + .backend + .send_request(backend_protocol::Lookup { millis: msg.millis }, deadline); + let work = async move { downstream?.await }; + if let Err(e) = ctx.pipe_reply( + JobOptions::new(), + token, + |_| work, + |res, token| Delegated { res, token }, + ) { + e.into_token().overloaded(); + } + } + + #[request_handler] + async fn ping(&mut self, _: router_protocol::Ping, _: &Context) -> &'static str { + "pong" + } + + #[send_handler] + async fn delegated(&mut self, msg: Delegated, _: &Context) { + match msg.res { + Ok(Ok(v)) => { + msg.token.reply(v); + } + _ => msg.token.overloaded(), + } + } +} + +// --------------------------------------------------------------------------- +// 2. CPU offload with a job deadline +// --------------------------------------------------------------------------- + +pub struct Verifier { + verdicts: Vec, +} +struct Verify { + id: u32, + spin: Duration, +} +impl Message for Verify { + type Result = (); +} +struct Verdict { + id: u32, + res: JobOutcome, +} +impl Message for Verdict { + type Result = (); +} +struct Verdicts; +impl Message for Verdicts { + type Result = Vec; +} + +#[actor] +impl Verifier { + #[send_handler] + async fn verify(&mut self, msg: Verify, ctx: &Context) { + // Reserve first: a refusal still has `msg` to report, so every + // object gets exactly one verdict. + let opts = JobOptions::new().deadline(Duration::from_millis(100)); + let permit = match ctx.reserve_blocking_job(opts) { + Ok(permit) => permit, + Err(e) => return self.verdicts.push(format!("{}: rejected ({e})", msg.id)), + }; + let (id, spin) = (msg.id, msg.spin); + // On Tokio's blocking pool, under the actor's native-job budget. + permit.pipe_blocking( + move |_job| { + std::thread::sleep(spin); // stands in for a CPU-bound check + true + }, + move |res| Verdict { id, res }, + ); + } + + #[send_handler] + async fn verdict(&mut self, msg: Verdict, _: &Context) { + // `JobOutcome` and `JobError` are `#[non_exhaustive]`. + self.verdicts.push(match msg.res { + Ok(valid) => format!("{}: valid={valid}", msg.id), + Err(JobError::DeadlineExceeded) => format!("{}: timed out", msg.id), + Err(JobError::Panicked(m)) => format!("{}: panicked {m}", msg.id), + Err(other) => format!("{}: {other}", msg.id), + }); + } + + #[request_handler] + async fn verdicts(&mut self, _: Verdicts, _: &Context) -> Vec { + std::mem::take(&mut self.verdicts) + } +} + +// --------------------------------------------------------------------------- +// 3. Outgoing requests by id (ethrex `outgoing_request`) +// --------------------------------------------------------------------------- + +pub struct Connection { + current_requests: PendingReplies, +} +struct Outgoing { + id: u64, +} +impl Message for Outgoing { + type Result = String; +} +struct PeerResponse { + id: u64, + body: String, +} +impl Message for PeerResponse { + type Result = bool; +} + +#[actor] +impl Connection { + #[deferred_handler] + async fn outgoing( + &mut self, + msg: Outgoing, + reply: PendingReply<'_, String>, + _: &Context, + ) { + let Ok(token) = reply.defer() else { return }; + // The request goes out on the socket on Ok(()). A refusal hands back + // its reason and the token together, so every reason is answered. + if let Err(refused) = self.current_requests.insert(msg.id, token) { + let (kind, token) = refused.into_parts(); + match kind { + InsertErrorKind::Duplicate => drop(token), // ReplyDropped; the first caller is kept + _ => token.overloaded(), // Full, and any reason added later + } + } + } + + /// Returns whether the response matched a pending request. + #[request_handler] + async fn peer_response(&mut self, msg: PeerResponse, _: &Context) -> bool { + match self.current_requests.take(&msg.id) { + Some(token) => { + token.reply(msg.body); + true + } + None => false, // late or unknown: expired, abandoned or purged + } + } +} + +// --------------------------------------------------------------------------- +// 4. Waiting for a free peer (ethrex `get_best_peer`) +// --------------------------------------------------------------------------- + +const MAX_WAITERS: usize = 2; + +pub struct PeerTable { + free: u32, + waiters: VecDeque>, +} +struct WaitForPeer; +impl Message for WaitForPeer { + type Result = u32; +} +struct Release(u32); +impl Message for Release { + type Result = (); +} + +#[actor] +impl PeerTable { + #[deferred_handler] + async fn wait_for_peer( + &mut self, + _: WaitForPeer, + reply: PendingReply<'_, u32>, + _: &Context, + ) { + if self.free > 0 { + self.free -= 1; + reply.reply(self.free); + return; + } + self.waiters.retain(|t| !t.is_closed()); + if self.waiters.len() >= MAX_WAITERS { + return reply.overloaded(); + } + // On `Err` (Overloaded, Closed or NoCaller) the caller was already + // answered, or there is none. + if let Ok(token) = reply.defer() { + self.waiters.push_back(token); + } + } + + #[send_handler] + async fn release(&mut self, msg: Release, _: &Context) { + self.free += msg.0; + while self.free > 0 { + let Some(token) = self.waiters.pop_front() else { + break; + }; + if token.is_closed() { + continue; + } + self.free -= 1; + token.reply(self.free); + } + } +} + +// --------------------------------------------------------------------------- +// 5. Owner stop with stored tokens +// --------------------------------------------------------------------------- + +pub struct Parker { + parked: Vec>, + answer_first: usize, +} +struct Park; +impl Message for Park { + type Result = &'static str; +} +struct StopAnswering(usize); +impl Message for StopAnswering { + type Result = (); +} + +#[actor] +impl Parker { + #[deferred_handler] + async fn park(&mut self, _: Park, reply: PendingReply<'_, &'static str>, _: &Context) { + if let Ok(token) = reply.defer() { + self.parked.push(token); + } + } + + /// Stop; `stopped()` answers the first `n` parked callers. + #[request_handler] + async fn stop_answering(&mut self, msg: StopAnswering, ctx: &Context) { + self.answer_first = msg.0; + ctx.stop(); + } + + #[stopped] + async fn on_stop(&mut self, _: &Context) { + let n = self.answer_first.min(self.parked.len()); + for token in self.parked.drain(..n) { + token.reply("answered in stopped()"); + } + } +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() { + // 1. Delegation: the slow request does not block the router. + let backend = Backend.start(); + let router = Router { + backend: backend.clone(), + } + .start(); + let slow = router.route(300); + assert_eq!(router.ping().await.unwrap(), "pong"); + assert_eq!(router.route(1).await.unwrap(), 1); + assert_eq!(slow.await.unwrap(), 300); + println!("1. router served other requests while a delegated one was pending"); + + // 2. CPU offload: a fast check, a check past its job deadline. + let verifier = Verifier { + verdicts: Vec::new(), + } + .start(); + verifier + .send(Verify { + id: 1, + spin: Duration::from_millis(1), + }) + .unwrap(); + verifier + .send(Verify { + id: 2, + spin: Duration::from_millis(400), + }) + .unwrap(); + let mut verdicts = Vec::new(); + while verdicts.len() < 2 { + verdicts.extend(verifier.request(Verdicts).await.unwrap()); + tokio::time::sleep(Duration::from_millis(10)).await; + } + verdicts.sort(); + assert_eq!(verdicts, ["1: valid=true", "2: timed out"]); + println!("2. {verdicts:?} (the hung check's thread keeps its slot until it returns)"); + + // 3. Outgoing requests by id. + let conn = Connection { + current_requests: PendingReplies::with_capacity(2), + } + .start(); + let a = conn + .send_request( + Outgoing { id: 1 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let dup = conn + .send_request( + Outgoing { id: 1 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let b = conn + .send_request( + Outgoing { id: 2 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + let full = conn + .send_request( + Outgoing { id: 3 }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ) + .unwrap(); + assert!(matches!(dup.await, Err(ActorError::ReplyDropped))); + assert!(matches!(full.await, Err(ActorError::Overloaded))); + drop(b); // the caller gives up: its entry is purged on the next insert + assert!(conn + .request(PeerResponse { + id: 1, + body: "block".into() + }) + .await + .unwrap()); + assert_eq!(a.await.unwrap(), "block"); + assert!(!conn + .request(PeerResponse { + id: 2, + body: "late".into() + }) + .await + .unwrap()); + println!("3. responses matched by id; duplicate, full and late responses handled"); + + // 4. Waiting for a free peer instead of polling. + let table = PeerTable { + free: 0, + waiters: VecDeque::new(), + } + .start(); + let first = table + .send_request(WaitForPeer, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + let second = table + .send_request(WaitForPeer, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); + assert!(matches!( + table.request(WaitForPeer).await, + Err(ActorError::Overloaded) + )); + table.send(Release(2)).unwrap(); + assert_eq!((first.await.unwrap(), second.await.unwrap()), (1, 0)); + println!("4. waiters answered when capacity freed; Overloaded at the queue bound"); + + // 5. Owner stop: stopped() answers one caller, death answers the other. + let parker = Parker { + parked: Vec::new(), + answer_first: 0, + } + .start(); + let one = parker.send_request(Park, Deadline::Never).unwrap(); + let two = parker.send_request(Park, Deadline::Never).unwrap(); + parker.request(StopAnswering(1)).await.unwrap(); + assert_eq!(one.await.unwrap(), "answered in stopped()"); + assert!(matches!( + two.await, + Err(ActorError::ActorStopped( + spawned_concurrency::ExitReason::Normal + )) + )); + parker.join().await; + println!("5. stopped() answered a stored token; death answered the rest ActorStopped(normal)"); + + for actor in [ + router.child_handle(), + backend.child_handle(), + verifier.child_handle(), + conn.child_handle(), + table.child_handle(), + ] { + actor.stop(); + } +} diff --git a/examples/capabilities/src/bin/execution_groups.rs b/examples/capabilities/src/bin/execution_groups.rs index eb0af643f..34aa0305e 100644 --- a/examples/capabilities/src/bin/execution_groups.rs +++ b/examples/capabilities/src/bin/execution_groups.rs @@ -12,15 +12,25 @@ impl Message for Double { struct Worker; impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, msg: Double, _: &Context) -> u64 { - msg.0 * 2 + async fn handle( + &mut self, + msg: Double, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(msg.0 * 2); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, msg: Double, _: &Context) -> u64 { - self.0.request(msg).await.unwrap() + async fn handle( + &mut self, + msg: Double, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0.request(msg).await.unwrap()); } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/execution_profiles.rs b/examples/capabilities/src/bin/execution_profiles.rs index 493b169d5..267671d8f 100644 --- a/examples/capabilities/src/bin/execution_profiles.rs +++ b/examples/capabilities/src/bin/execution_profiles.rs @@ -12,11 +12,16 @@ impl Message for Location { type Result = (String, Option); } impl Handler for Worker { - async fn handle(&mut self, _: Location, _: &Context) -> ::Result { - ( + async fn handle( + &mut self, + _: Location, + reply: spawned_concurrency::PendingReply<'_, ::Result>, + _: &Context, + ) { + reply.reply(( std::thread::current().name().unwrap_or_default().to_owned(), spawned_rt::tasks::affinity::current_cpu().ok(), - ) + )); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/exit_signals.rs b/examples/capabilities/src/bin/exit_signals.rs new file mode 100644 index 000000000..807673ab1 --- /dev/null +++ b/examples/capabilities/src/bin/exit_signals.rs @@ -0,0 +1,246 @@ +//! Exit signals as in OTP (docs/GUIDE.md, "Exit signals"): `exit` vs `kill`, +//! trapping, `NoProc` for registrations and sends after death, and +//! `exit_pending` for work queued before a supervisor's shutdown. +use spawned_concurrency::{ + message::Message, + reply::PendingReply, + supervision::{ChildSpec, SupervisorConfig}, + system::ActorSystem, + tasks::{Actor, ActorStart, Context, Handler}, + ActorError, ChildHandle, Down, Exit, ExitReason, NoProc, +}; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; + +/// What an actor observed. +#[derive(Clone, Default)] +struct Seen(Arc>>); +impl Seen { + fn push(&self, entry: String) { + self.0.lock().unwrap().push(entry); + } + fn take(&self) -> Vec { + std::mem::take(&mut self.0.lock().unwrap()) + } +} + +struct Worker { + trap: bool, + seen: Seen, +} +impl Worker { + fn new(trap: bool, seen: &Seen) -> Self { + Self { + trap, + seen: seen.clone(), + } + } +} +impl Actor for Worker { + async fn started(&mut self, ctx: &Context) { + // Trap first thing: signals that arrive during started() are + // resolved with the flag as it is when started() returns. + ctx.trap_exit(self.trap); + } + async fn stopped(&mut self, ctx: &Context) { + self.seen + .push(format!("stopped (exit_pending: {})", ctx.exit_pending())); + } + async fn exit_received(&mut self, exit: Exit, _: &Context) { + self.seen.push(format!("Exit {:?}", exit.reason)); + } +} +impl Handler for Worker { + async fn handle(&mut self, down: Down, reply: PendingReply<'_, ()>, _: &Context) { + self.seen.push(format!("Down {:?}", down.reason)); + reply.reply(()); + } +} + +/// Link to and monitor a peer. +struct Watch(ChildHandle); +impl Message for Watch { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, msg: Watch, reply: PendingReply<'_, ()>, ctx: &Context) { + ctx.link(&msg.0); + ctx.monitor(&msg.0); + reply.reply(()); + } +} + +/// `try_link`: `Err(NoProc)` instead of a `NoProc` exit signal. +struct TryLink(ChildHandle); +impl Message for TryLink { + type Result = Result<(), NoProc>; +} +impl Handler for Worker { + async fn handle( + &mut self, + msg: TryLink, + reply: PendingReply<'_, Result<(), NoProc>>, + ctx: &Context, + ) { + reply.reply(ctx.try_link(&msg.0)); + } +} + +/// Work that refuses to start once the actor's exit is pending. +struct Work(u32); +impl Message for Work { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, msg: Work, reply: PendingReply<'_, ()>, ctx: &Context) { + let pending = ctx.exit_pending(); + self.seen + .push(format!("work {} (exit_pending: {pending})", msg.0)); + reply.reply(()); + } +} + +/// Busy until the actor's exit is pending (bounded), standing in for a long +/// handler that is running when the supervisor shuts down. +struct Busy; +impl Message for Busy { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, _: Busy, reply: PendingReply<'_, ()>, ctx: &Context) { + for _ in 0..1000 { + if ctx.exit_pending() { + break; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + reply.reply(()); + } +} + +/// A round trip: everything queued before it was handled. +struct Sync; +impl Message for Sync { + type Result = (); +} +impl Handler for Worker { + async fn handle(&mut self, _: Sync, reply: PendingReply<'_, ()>, _: &Context) { + reply.reply(()); + } +} + +const LIMIT: Duration = Duration::from_secs(5); + +async fn ended(handle: &ChildHandle) -> ExitReason { + tokio::time::timeout(LIMIT, handle.wait_exit_async()) + .await + .expect("actor did not exit") +} + +#[tokio::main(flavor = "multi_thread", worker_threads = 2)] +async fn main() { + // 1. exit(reason) follows OTP's exit_signal/2: Normal is ignored by a + // non-trapping actor, any other reason ends it without stopped(). + let seen = Seen::default(); + let plain = Worker::new(false, &seen).start(); + plain.request(Sync).await.unwrap(); + plain.child_handle().exit(ExitReason::Normal); + plain.request(Sync).await.unwrap(); + assert!(plain.child_handle().is_alive(), "exit(Normal) is ignored"); + plain.child_handle().exit(ExitReason::Shutdown); + assert_eq!(ended(&plain.child_handle()).await, ExitReason::Shutdown); + assert!( + seen.take().is_empty(), + "an untrapped signal skips stopped()" + ); + println!("exit(Normal) ignored; exit(Shutdown) ended the actor without stopped()"); + + // 2. A trapping actor receives exit signals as `Exit` messages; only a + // kill is untrappable and ends it `Killed`, also without stopped(). + // An administrative stop (`stop`, `shutdown`) always runs stopped(). + let (observer_seen, target_seen) = (Seen::default(), Seen::default()); + let observer = Worker::new(true, &observer_seen).start(); + let target = Worker::new(true, &target_seen).start(); + observer + .request(Watch(target.child_handle())) + .await + .unwrap(); + target.child_handle().exit(ExitReason::Shutdown); + target.request(Sync).await.unwrap(); + assert_eq!(target_seen.take(), vec!["Exit Shutdown"]); + target.child_handle().kill(); + assert_eq!(ended(&target.child_handle()).await, ExitReason::Killed); + assert!(target_seen.take().is_empty(), "a kill skips stopped()"); + // The trapping observer gets the link notice and the monitor's Down. + tokio::time::sleep(Duration::from_millis(50)).await; + observer.request(Sync).await.unwrap(); + let mut notices = observer_seen.take(); + notices.sort(); + assert_eq!(notices, vec!["Down Killed", "Exit Killed"]); + println!("trapped Exit(Shutdown); kill() ended a trapping actor Killed; peers saw Killed"); + + // 3. Past its death point an actor is gone: link and monitor get NoProc + // (one notice each), try_link returns Err(NoProc), sends fail NoProc. + observer + .request(Watch(target.child_handle())) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + observer.request(Sync).await.unwrap(); + let mut notices = observer_seen.take(); + notices.sort(); + assert_eq!(notices, vec!["Down NoProc", "Exit NoProc"]); + let tried = observer.request(TryLink(target.child_handle())).await; + assert_eq!(tried.unwrap(), Err(NoProc)); + assert!(matches!(target.send(Sync), Err(ActorError::NoProc))); + // The real reason is still available from the handle. + assert_eq!( + target.child_handle().exit_reason(), + Some(ExitReason::Killed) + ); + // A non-trapping actor that links to a dead one exits NoProc itself. + let linker = Worker::new(false, &Seen::default()).start(); + linker.request(Watch(target.child_handle())).await.unwrap(); + assert_eq!(ended(&linker.child_handle()).await, ExitReason::NoProc); + observer.child_handle().stop(); + ended(&observer.child_handle()).await; + assert_eq!(observer_seen.take(), vec!["stopped (exit_pending: true)"]); + println!("late link/monitor gave NoProc, try_link Err(NoProc), send NoProc"); + + // 4. A supervisor's shutdown is its parent's exit signal. A trapping + // child handles the work queued before it, sees `exit_pending()`, then + // runs stopped(); a non-trapping child would end without stopped(). + let system = ActorSystem::new().unwrap(); + let group = system.async_group("exit-signals", 1).unwrap(); + let mut tree = system.supervisor(SupervisorConfig::default()).unwrap(); + let child_seen = Seen::default(); + let s = child_seen.clone(); + let child = tree + .add(ChildSpec::new("trapping", &group, move |_| { + let seen = s.clone(); + async move { Ok::<_, String>(Worker::new(true, &seen)) } + })) + .unwrap(); + let supervisor = tree.start().await.unwrap(); + let worker = child.current().unwrap(); + worker.request(Work(1)).await.unwrap(); + // Work queued behind a busy handler, then the shutdown: the parent exit + // queues behind that work. + worker.send(Busy).unwrap(); + worker.send(Work(2)).unwrap(); + let handle = worker.child_handle(); + assert!(supervisor.shutdown(LIMIT).await.complete); + assert_eq!(ended(&handle).await, ExitReason::Shutdown); + assert_eq!( + child_seen.take(), + vec![ + "work 1 (exit_pending: false)", + "work 2 (exit_pending: true)", + "stopped (exit_pending: true)", + ] + ); + assert!(system.shutdown(LIMIT).await.complete); + println!("supervisor shutdown: queued work saw exit_pending, then stopped() ran"); +} diff --git a/examples/capabilities/src/bin/framework_metrics.rs b/examples/capabilities/src/bin/framework_metrics.rs index 089e7c47f..2e1d663b5 100644 --- a/examples/capabilities/src/bin/framework_metrics.rs +++ b/examples/capabilities/src/bin/framework_metrics.rs @@ -11,14 +11,18 @@ impl Message for Ping { } impl Actor for Worker {} impl Handler for Worker { - async fn handle(&mut self, _: Ping, _: &Context) {} + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] async fn main() { - metrics::configure(Config { - timing_sample_every: 16, - }) - .unwrap(); + metrics::configure(Config::default().timing_sample_every(16)).unwrap(); let actor = Worker.start(); for _ in 0..32 { actor.request(Ping).await.unwrap(); diff --git a/examples/capabilities/src/bin/graceful_drain.rs b/examples/capabilities/src/bin/graceful_drain.rs index 022902d40..139f93cc1 100644 --- a/examples/capabilities/src/bin/graceful_drain.rs +++ b/examples/capabilities/src/bin/graceful_drain.rs @@ -10,8 +10,14 @@ impl Message for Add { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Add, _: &Context) { + async fn handle( + &mut self, + _: Add, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0 += 1; + reply.reply(()); } } struct Fence; @@ -19,8 +25,13 @@ impl Message for Fence { type Result = usize; } impl Handler for Worker { - async fn handle(&mut self, _: Fence, _: &Context) -> usize { - self.0 + async fn handle( + &mut self, + _: Fence, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.0); } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] diff --git a/examples/capabilities/src/bin/native_lifecycle.rs b/examples/capabilities/src/bin/native_lifecycle.rs index 5772fee23..b371eae83 100644 --- a/examples/capabilities/src/bin/native_lifecycle.rs +++ b/examples/capabilities/src/bin/native_lifecycle.rs @@ -19,9 +19,15 @@ impl Actor for Observer { } } impl Handler for Observer { - fn handle(&mut self, event: Down, _: &Context) { + fn handle( + &mut self, + event: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.counts.1 += 1; self.downs.send(event).unwrap(); + reply.reply(()); } } struct Counts; @@ -29,8 +35,13 @@ impl Message for Counts { type Result = (usize, usize); } impl Handler for Observer { - fn handle(&mut self, _: Counts, _: &Context) -> (usize, usize) { - self.counts + fn handle( + &mut self, + _: Counts, + reply: spawned_concurrency::PendingReply<'_, (usize, usize)>, + _: &Context, + ) { + reply.reply(self.counts); } } struct Failing; diff --git a/examples/capabilities/src/bin/protocol_timeouts.rs b/examples/capabilities/src/bin/protocol_timeouts.rs index f62b08fac..da788a389 100644 --- a/examples/capabilities/src/bin/protocol_timeouts.rs +++ b/examples/capabilities/src/bin/protocol_timeouts.rs @@ -23,15 +23,31 @@ mod asynchronous { } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Value, _: &Context) -> u64 { - self.1.fetch_add(1, Ordering::SeqCst); - 42 + async fn handle( + &mut self, + _: lookup_protocol::Value, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; + reply.reply(__reply_value); } } impl Handler for Worker { - async fn handle(&mut self, _: lookup_protocol::Quick, _: &Context) -> u64 { - self.1.fetch_add(1, Ordering::SeqCst); - 42 + async fn handle( + &mut self, + _: lookup_protocol::Quick, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; + reply.reply(__reply_value); } } pub async fn run() { @@ -63,15 +79,31 @@ mod synchronous { } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Value, _: &Context) -> u64 { - self.1.fetch_add(1, Ordering::SeqCst); - 42 + fn handle( + &mut self, + _: lookup_protocol::Value, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; + reply.reply(__reply_value); } } impl Handler for Worker { - fn handle(&mut self, _: lookup_protocol::Quick, _: &Context) -> u64 { - self.1.fetch_add(1, Ordering::SeqCst); - 42 + fn handle( + &mut self, + _: lookup_protocol::Quick, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.1.fetch_add(1, Ordering::SeqCst); + 42 + }; + reply.reply(__reply_value); } } pub fn run() { diff --git a/examples/capabilities/src/bin/registrations_and_fakes.rs b/examples/capabilities/src/bin/registrations_and_fakes.rs index 7becca84c..b0ae7fd6e 100644 --- a/examples/capabilities/src/bin/registrations_and_fakes.rs +++ b/examples/capabilities/src/bin/registrations_and_fakes.rs @@ -37,8 +37,15 @@ impl spawned_concurrency::message::Message for Crash { type Result = (); } impl Handler for Service { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("intentional named-service replacement"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("intentional named-service replacement"); + } } } #[tokio::main(flavor = "current_thread")] @@ -49,15 +56,14 @@ async fn main() { let system = ActorSystem::new().unwrap(); let group = system.async_group("named-service", 1).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let service = tree .add(ChildSpec::new("prices", &group, |_| async { diff --git a/examples/capabilities/src/bin/runtime_isolation.rs b/examples/capabilities/src/bin/runtime_isolation.rs index b0b39c801..3dad79d33 100644 --- a/examples/capabilities/src/bin/runtime_isolation.rs +++ b/examples/capabilities/src/bin/runtime_isolation.rs @@ -11,16 +11,29 @@ impl Message for Ping { } impl Actor for Critical {} impl Handler for Critical { - async fn handle(&mut self, _: Ping, _: &Context) -> String { - std::thread::current().name().unwrap().to_owned() + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, String>, + _: &Context, + ) { + reply.reply(std::thread::current().name().unwrap().to_owned()); } } struct Router(ActorRef); impl Actor for Router {} impl Handler for Router { - async fn handle(&mut self, _: Ping, _: &Context) -> String { - assert_eq!(std::thread::current().name(), Some("background-worker")); - self.0.request(Ping).await.unwrap() + async fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, String>, + _: &Context, + ) { + let __reply_value: String = { + assert_eq!(std::thread::current().name(), Some("background-worker")); + self.0.request(Ping).await.unwrap() + }; + reply.reply(__reply_value); } } diff --git a/examples/capabilities/src/bin/shutdown_diagnostics.rs b/examples/capabilities/src/bin/shutdown_diagnostics.rs index cda151fef..b8470c3e0 100644 --- a/examples/capabilities/src/bin/shutdown_diagnostics.rs +++ b/examples/capabilities/src/bin/shutdown_diagnostics.rs @@ -16,9 +16,15 @@ impl Message for Hold { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, msg: Hold, _: &Context) { + async fn handle( + &mut self, + msg: Hold, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { let _ = msg.0.send(()); let _ = msg.1.await; + reply.reply(()); } } struct Unhealthy; @@ -26,9 +32,15 @@ impl Message for Unhealthy { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Unhealthy, ctx: &Context) { + async fn handle( + &mut self, + _: Unhealthy, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { // Cooperative abnormal exit, without a panic or forced interruption. ctx.stop_with_reason(ExitReason::Panic("reported unhealthy worker".into())); + reply.reply(()); } } diff --git a/examples/capabilities/src/bin/streams_and_recipients.rs b/examples/capabilities/src/bin/streams_and_recipients.rs index d9a84b14e..919dde467 100644 --- a/examples/capabilities/src/bin/streams_and_recipients.rs +++ b/examples/capabilities/src/bin/streams_and_recipients.rs @@ -19,13 +19,24 @@ mod asynchronous { use spawned_concurrency::tasks::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, msg: Item, _: &Context) { + async fn handle( + &mut self, + msg: Item, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0 += N * msg.0; + reply.reply(()); } } impl Handler for Counter { - async fn handle(&mut self, _: Total, _: &Context) -> u64 { - self.0 + async fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0); } } pub async fn run() { @@ -58,7 +69,7 @@ mod asynchronous { .expect("async listener panicked"); assert!(matches!( recipients[0].send(Item(1)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); recipients[1].send(Item(1)).unwrap(); assert_eq!(second.request(Total).await.unwrap(), 4); @@ -73,13 +84,24 @@ mod synchronous { use spawned_concurrency::threads::{self, Actor, ActorStart, Context, Handler, Recipient}; impl Actor for Counter {} impl Handler for Counter { - fn handle(&mut self, msg: Item, _: &Context) { + fn handle( + &mut self, + msg: Item, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0 += N * msg.0; + reply.reply(()); } } impl Handler for Counter { - fn handle(&mut self, _: Total, _: &Context) -> u64 { - self.0 + fn handle( + &mut self, + _: Total, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + reply.reply(self.0); } } struct Source { @@ -140,7 +162,7 @@ mod synchronous { .expect("native counter did not stop"); assert!(matches!( recipients[0].send(Item(1)), - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) )); recipients[1].send(Item(1)).unwrap(); assert_eq!(second.request(Total).unwrap(), 4); diff --git a/examples/capabilities/src/bin/supervision_recovery.rs b/examples/capabilities/src/bin/supervision_recovery.rs index 967dc7117..288ec59b6 100644 --- a/examples/capabilities/src/bin/supervision_recovery.rs +++ b/examples/capabilities/src/bin/supervision_recovery.rs @@ -17,14 +17,29 @@ impl Message for Crash { struct Counter(u64); impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Next, _: &Context) -> u64 { - self.0 += 1; - self.0 + async fn handle( + &mut self, + _: Next, + reply: spawned_concurrency::PendingReply<'_, u64>, + _: &Context, + ) { + let __reply_value: u64 = { + self.0 += 1; + self.0 + }; + reply.reply(__reply_value); } } impl Handler for Counter { - async fn handle(&mut self, _: Crash, _: &Context) { - panic!("intentional recovery example failure"); + async fn handle( + &mut self, + _: Crash, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("intentional recovery example failure"); + } } } #[tokio::main(flavor = "current_thread")] diff --git a/examples/capabilities/src/bin/supervision_tree.rs b/examples/capabilities/src/bin/supervision_tree.rs index 4a6bf2d69..82859127e 100644 --- a/examples/capabilities/src/bin/supervision_tree.rs +++ b/examples/capabilities/src/bin/supervision_tree.rs @@ -19,8 +19,15 @@ impl Message for Fail { type Result = (); } impl Handler for Worker { - async fn handle(&mut self, _: Fail, _: &Context) { - panic!("example failure"); + async fn handle( + &mut self, + _: Fail, + _reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + panic!("example failure"); + } } } #[tokio::main(flavor = "multi_thread", worker_threads = 2)] @@ -33,11 +40,11 @@ async fn main() { let controls = system.async_group("tree-control", 1).unwrap(); let workers = system.thread_group("tree-workers", 3).unwrap(); let mut tree = system - .supervisor(SupervisorConfig { - strategy, - max_restarts: 20, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default() + .strategy(strategy) + .max_restarts(20), + ) .unwrap(); let earlier = tree .add(ChildSpec::new("earlier", &workers, |_| async { @@ -51,10 +58,7 @@ async fn main() { .add_supervisor(SupervisorSpec::new( "subsystem", &controls, - SupervisorConfig { - max_restarts: 0, - ..SupervisorConfig::default() - }, + SupervisorConfig::default().max_restarts(0), move |subtree| { let child = subtree.add(ChildSpec::new("leaf", &placement, |_| async { Ok::<_, String>(Worker) diff --git a/examples/capabilities/src/bin/test_kit.rs b/examples/capabilities/src/bin/test_kit.rs index 20ad7467a..f8151eb22 100644 --- a/examples/capabilities/src/bin/test_kit.rs +++ b/examples/capabilities/src/bin/test_kit.rs @@ -110,15 +110,14 @@ async fn main() { let test = kit::TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let rates_ref = rates.to_rates_ref(); let probe = out.clone(); diff --git a/examples/capabilities/src/bin/timeout_outcomes.rs b/examples/capabilities/src/bin/timeout_outcomes.rs index ac10dd20b..fcf745a96 100644 --- a/examples/capabilities/src/bin/timeout_outcomes.rs +++ b/examples/capabilities/src/bin/timeout_outcomes.rs @@ -2,7 +2,7 @@ use spawned_concurrency::{ message::Message, tasks::{Actor, ActorStart, Context, Handler}, - ActorError, Response, + ActorError, Deadline, Response, }; use std::time::Duration; use tokio::sync::oneshot; @@ -17,10 +17,18 @@ impl Message for Apply { type Result = (); } impl Handler for Counter { - async fn handle(&mut self, msg: Apply, _: &Context) { - self.0 += 1; // Effect happens before the reply. - msg.entered.send(()).unwrap(); - let _ = msg.release.await; + async fn handle( + &mut self, + msg: Apply, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + { + self.0 += 1; // Effect happens before the reply. + msg.entered.send(()).unwrap(); + let _ = msg.release.await; + } + reply.reply(()); } } struct Read; @@ -28,8 +36,13 @@ impl Message for Read { type Result = usize; } impl Handler for Counter { - async fn handle(&mut self, _: Read, _: &Context) -> usize { - self.0 + async fn handle( + &mut self, + _: Read, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.0); } } @@ -38,13 +51,18 @@ async fn main() { let counter = Counter(0).start(); let (entered_tx, entered_rx) = oneshot::channel(); let (release_tx, release_rx) = oneshot::channel(); - let reply = counter.request_raw(Apply { - entered: entered_tx, - release: release_rx, - }); + // The deadline is fixed at send (OTP's call/3): already due here. An + // expired request is still dispatched, as in OTP; its late reply is dropped. + let reply = counter.send_request( + Apply { + entered: entered_tx, + release: release_rx, + }, + Deadline::after(Duration::ZERO), + ); entered_rx.await.unwrap(); // A gate keeps the reply pending; no race-dependent sleep is needed. - let result = Response::from_with_timeout(reply, Duration::ZERO).await; + let result = Response::from(reply).await; assert!(matches!(result, Err(ActorError::RequestTimeout))); release_tx.send(()).unwrap(); assert_eq!(counter.request(Read).await.unwrap(), 1); diff --git a/examples/exit_reason/src/main.rs b/examples/exit_reason/src/main.rs index e5e8f9357..865c55b79 100644 --- a/examples/exit_reason/src/main.rs +++ b/examples/exit_reason/src/main.rs @@ -94,7 +94,14 @@ fn main() { } } impl Handler for PanicOnStart { - async fn handle(&mut self, _msg: Noop, _ctx: &Context) {} + async fn handle( + &mut self, + _msg: Noop, + reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { + reply.reply(()); + } } let worker3 = PanicOnStart.start(); let reason = worker3.wait_exit().await; @@ -170,23 +177,40 @@ fn main() { } impl Actor for Observer {} impl Handler for Observer { - async fn handle(&mut self, msg: StartMonitor, ctx: &Context) -> MonitorRef { - ctx.monitor(&msg.0) + async fn handle( + &mut self, + msg: StartMonitor, + reply: spawned_concurrency::PendingReply<'_, MonitorRef>, + ctx: &Context, + ) { + reply.reply(ctx.monitor(&msg.0)); } } impl Handler for Observer { - async fn handle(&mut self, msg: StopMonitor, ctx: &Context) { + async fn handle( + &mut self, + msg: StopMonitor, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.demonitor(msg.0); + reply.reply(()); } } impl Handler for Observer { - async fn handle(&mut self, msg: Down, _ctx: &Context) { + async fn handle( + &mut self, + msg: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { tracing::info!( "[observer] received Down for {} ({})", msg.monitor_ref, msg.reason ); self.log.lock().unwrap().push(msg); + reply.reply(()); } } @@ -259,8 +283,14 @@ fn main() { } } impl Handler for Supervisor { - async fn handle(&mut self, msg: LinkChild, ctx: &Context) { + async fn handle( + &mut self, + msg: LinkChild, + reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { ctx.link(&msg.0); + reply.reply(()); } } diff --git a/examples/thread_only/src/main.rs b/examples/thread_only/src/main.rs index a8226371f..483b9663e 100644 --- a/examples/thread_only/src/main.rs +++ b/examples/thread_only/src/main.rs @@ -32,13 +32,25 @@ impl spawned_concurrency::message::Message for Tick { struct Observer(Arc); impl Actor for Observer {} impl Handler for Observer { - fn handle(&mut self, _: Down, _: &Context) { + fn handle( + &mut self, + _: Down, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } impl Handler for Observer { - fn handle(&mut self, _: Tick, _: &Context) { + fn handle( + &mut self, + _: Tick, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } fn main() { @@ -63,7 +75,9 @@ fn main() { let observer = Observer(events.clone()).start(); observer.context().monitor(&counter.child_handle()); spawned_concurrency::threads::send_after(Duration::from_millis(5), observer.context(), Tick); - counter.child_handle().shutdown(); + counter + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(counter.wait_exit(), ExitReason::Shutdown); assert_eq!( spawned_rt::threads::block_on(counter.child_handle().wait_exit_async()), @@ -84,7 +98,9 @@ fn main() { let first = Counter(0).start(); let second = Counter(0).start(); first.context().link(&second.child_handle()); - second.child_handle().shutdown(); + second + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(first.wait_exit(), ExitReason::Shutdown); assert_eq!(second.wait_exit(), ExitReason::Shutdown); #[cfg(feature = "metrics")] diff --git a/fuzz/regressions/cohort_rounds_charged_at_selection.input b/fuzz/regressions/cohort_rounds_charged_at_selection.input new file mode 100644 index 000000000..74a07fdf1 Binary files /dev/null and b/fuzz/regressions/cohort_rounds_charged_at_selection.input differ diff --git a/fuzz/regressions/late_link_noproc_exit.input b/fuzz/regressions/late_link_noproc_exit.input new file mode 100644 index 000000000..bab1ed3a1 Binary files /dev/null and b/fuzz/regressions/late_link_noproc_exit.input differ diff --git a/fuzz/regressions/late_link_noproc_exit.tape b/fuzz/regressions/late_link_noproc_exit.tape new file mode 100644 index 000000000..24dc55353 --- /dev/null +++ b/fuzz/regressions/late_link_noproc_exit.tape @@ -0,0 +1,137 @@ +# spawned-sim tape v2 +seed 7466577281530464928 +spin 10000 +c 0 +c 1 +c 2 +c 0 +c 3 +c 0 +t 1000000 +c 0 +c 3 +c 0 +t 2000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 3000000 +c 0 +c 3 +c 0 +t 4000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 5000000 +c 0 +c 3 +c 0 +t 6000000 +c 0 +c 3 +c 0 +c 3 +c 0 +t 7000000 +c 0 +c 3 +c 0 +t 8000000 +c 0 +c 3 +c 0 +p 5 +p 3 +p 4 +c 0 +c 6 +p 7 +c 0 +c 3 +c 0 +t 9000000 +c 0 +c 3 +c 0 +t 10000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 11000000 +c 0 +c 3 +c 0 +t 12000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 13000000 +c 0 +c 3 +c 0 +t 14000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +t 15000000 +c 0 +c 3 +c 0 +t 16000000 +c 0 +c 3 +c 0 +c 3 +c 0 +c 3 +c 0 +c 8 +c 0 +c 9 +c 0 +t 17000000 +c 0 +c 3 +c 0 +c 9 +c 0 +t 18000000 +c 0 +c 3 +c 0 +c 9 +c 0 +c 3 +c 0 +c 5 +c 0 +c 6 +c 7 +c 6 +c 0 +c 10 +c 3 +t 2768000000 +c 3 +c 10 +c 1 +c 0 +c 9 +c 0 diff --git a/fuzz/regressions/link_from_an_exiting_actor.input b/fuzz/regressions/link_from_an_exiting_actor.input new file mode 100644 index 000000000..5febc85c7 Binary files /dev/null and b/fuzz/regressions/link_from_an_exiting_actor.input differ diff --git a/fuzz/regressions/link_from_an_exiting_actor.tape b/fuzz/regressions/link_from_an_exiting_actor.tape new file mode 100644 index 000000000..dd2680c9f --- /dev/null +++ b/fuzz/regressions/link_from_an_exiting_actor.tape @@ -0,0 +1,46 @@ +# spawned-sim tape v2 +seed 2531906049332685619 +spin 10000 +c 0 +p 4 +p 5 +p 2 +p 3 +p 1 +c 0 +p 6 +c 3 +c 0 +c 4 +c 0 +c 6 +c 0 +t 1000000 +c 0 +c 2 +c 0 +c 3 +c 0 +c 4 +c 0 +c 6 +c 0 +t 2000000 +c 0 +c 2 +c 0 +c 3 +c 0 +c 4 +c 0 +c 6 +c 0 +c 7 +c 1 +c 0 +c 2 +p 6 +p 3 +c 0 +c 4 +c 0 diff --git a/fuzz/regressions/panic_then_late_shutdown_notice.input b/fuzz/regressions/panic_then_late_shutdown_notice.input new file mode 100644 index 000000000..5a8357ea4 Binary files /dev/null and b/fuzz/regressions/panic_then_late_shutdown_notice.input differ diff --git a/fuzz/src/driver.rs b/fuzz/src/driver.rs index 66aab6721..e6d661752 100644 --- a/fuzz/src/driver.rs +++ b/fuzz/src/driver.rs @@ -274,26 +274,25 @@ fn restart_mode(v: u8) -> RestartMode { fn config(p: &Policy, seed: u64) -> SupervisorConfig { let initial = Duration::from_millis(p.backoff as u64); - SupervisorConfig { - strategy: match p.strategy % 3 { + SupervisorConfig::default() + .strategy(match p.strategy % 3 { 0 => SupervisorStrategy::OneForOne, 1 => SupervisorStrategy::OneForAll, _ => SupervisorStrategy::RestForOne, - }, - max_restarts: (p.max_restarts % 7) as usize, + }) + .max_restarts((p.max_restarts % 7) as usize) // At least 50 ms, so a fast restart loop exhausts the budget // instead of restarting forever as virtual time creeps. - restart_window: Duration::from_millis(50 + 2 * p.window as u64), - backoff: Backoff { - initial, - maximum: initial * 4, - stable_after: Duration::from_millis(10 * p.stable as u64 + 1), - jitter: p.jitter, - // Never the default (wall-clock) jitter seed. - seed: Some(seed), - }, - ..SupervisorConfig::default() - } + .restart_window(Duration::from_millis(50 + 2 * p.window as u64)) + .backoff( + Backoff::default() + .initial(initial) + .maximum(initial * 4) + .stable_after(Duration::from_millis(10 * p.stable as u64 + 1)) + .jitter(p.jitter) + // Never the default (wall-clock) jitter seed. + .seed(seed), + ) } fn cfg_strategy(p: &Policy) -> SupervisorStrategy { @@ -306,6 +305,7 @@ fn new_sup_inst(world: &World, root: usize, cfg: &SupervisorConfig) -> usize { root, max_restarts: cfg.max_restarts, window: cfg.restart_window, + cohort: !matches!(cfg.strategy, SupervisorStrategy::OneForOne), strategy: cfg.strategy, backoff: cfg.backoff.clone(), flat: false, @@ -559,7 +559,8 @@ impl Driver { ); } } - Err(ActorError::ActorStopped) => {} + // Answered at the death point, or sent after it (0.8: NoProc). + Err(ActorError::ActorStopped(_) | ActorError::NoProc) => {} #[allow(unreachable_patterns)] Err(other) => self.violate("R1: unexpected request error", format!("{other}")), } @@ -646,10 +647,14 @@ impl Driver { }; self.may_stop(pa.id()); self.may_stop(pb.id()); - // A context whose actor already exited cannot link: the call - // is a no-op (`register_link` sees its own table closed), so - // nothing is predicted for the pair. - let linker_exited = pa.child_handle().exit_reason().is_some(); + // A context whose actor already exited, or whose exit began, + // cannot link (0.8, as in OTP: an exiting process cannot call + // link): the call is inert, so nothing is predicted for the + // pair. (`exit_pending` also counts a queued parent exit, which + // does not make the link inert: the check is then skipped, + // conservatively.) + let linker_exited = + pa.child_handle().exit_reason().is_some() || pa.context().exit_pending(); pa.context().link(&pb.child_handle()); let trap = |d: &Self, id| d.traps.get(&id).is_some_and(|t| t.0); self.links.push(LinkRec { @@ -737,7 +742,7 @@ impl Driver { } match r { Reason::Normal => handle.stop(), - Reason::Shutdown => handle.shutdown(), + Reason::Shutdown => handle.stop_with_reason(ExitReason::Shutdown), r => handle.stop_with_reason(reason(r)), } } @@ -830,7 +835,7 @@ impl Driver { timer: 0, act, }; - drop(aref.request_raw(msg)); + drop(aref.send_request(msg, spawned_concurrency::DEFAULT_REQUEST_TIMEOUT)); } Op::WaitGeneration { sup, child, ms } => self.wait_generation(sup, child, ms).await, Op::Fence => { @@ -1556,7 +1561,7 @@ impl Driver { } true } - Err(ActorError::ActorStopped) => { + Err(ActorError::ActorStopped(_) | ActorError::NoProc) => { if rt::timeout(LONG, aref.wait_exit()).await.is_err() { self.violate( "L7: a stopping actor never published its exit", @@ -1659,7 +1664,9 @@ impl Driver { .collect() }; for r in received(self) { - if r != published { + // 0.8 (X1): a monitor installed after the target's death + // point gets `NoProc`; one installed before, its reason. + if r != published && r != ExitReason::NoProc { self.violate( "X1: Down carried a different reason than the target published", format!("{r:?} vs {published:?}"), @@ -1743,7 +1750,9 @@ impl Driver { .unwrap_or_default() }; for r in exits(self) { - if r != published { + // 0.8 (X3): a link made after the peer's death point is a + // late link, whose notice is `NoProc`. + if r != published && r != ExitReason::NoProc { self.violate( "X3: Exit carried a different reason than the peer published", format!("{r:?} vs {published:?}"), @@ -1804,7 +1813,14 @@ impl Driver { format!("w{instance} {exit:?}"), ); } - if panicked && !exit.is_abnormal() { + // A panic sets the reason, but until the death point an + // untrapped exit signal or link notice still replaces it + // (semantics.md L4, `Lifecycle!LateSignalWins`; OTP: a signal + // during `terminate` kills with its own reason). The only + // non-abnormal reason a signal can carry is `Shutdown`. + let trapping = self.traps.get(&id).is_some_and(|t| t.0); + let late_shutdown = exit == ExitReason::Shutdown && !trapping; + if panicked && !exit.is_abnormal() && !late_shutdown { self.violate( "L4: an actor that panicked exited with a non-abnormal reason", format!("w{instance} {exit:?}"), @@ -1855,6 +1871,12 @@ impl Driver { } let detail = format!("w{} published {exit:?}, peer {peer:?}", a.instance); drop(s); + // 0.8 (E4): a link made after the peer's death point is a late + // link; a non-trapping caller gets the `NoProc` exit signal and ends + // `NoProc`, whatever the peer's reason was. + if exit == ExitReason::NoProc { + return; + } if *peer == ExitReason::Normal { self.violate( "X3: a non-trapping actor was stopped by a linked peer's Normal exit", @@ -2009,7 +2031,7 @@ fn budget_checks(world: &World) { let calls = s.calls.clone(); let mut found = Vec::new(); let mut seen: HashMap<(usize, String), usize> = HashMap::new(); - let mut retries: Vec> = vec![Vec::new(); s.sups.len()]; + let mut retries: Vec> = vec![Vec::new(); s.sups.len()]; for c in &calls { let n = seen.entry((c.inst, c.slot.clone())).or_default(); *n += 1; @@ -2020,21 +2042,69 @@ fn budget_checks(world: &World) { c.slot.clone(), )); } - retries[c.inst].push(c.at); + retries[c.inst].push((c.at, &c.slot)); } } - for (inst, times) in retries.iter().enumerate() { + for (inst, log) in retries.iter().enumerate() { let SupInst { max_restarts, window, + cohort, .. } = s.sups[inst].clone(); - for (i, t) in times.iter().enumerate() { - let in_window = times[..=i].iter().filter(|u| *t - **u < window).count(); - if in_window > max_restarts { + if !cohort { + // A OneForOne retry is one action, stamped at its admission: + // the factory call's time is the grant's. + for (i, (t, _)) in log.iter().enumerate() { + let in_window = log[..=i].iter().filter(|(u, _)| *t - *u < window).count(); + if in_window > max_restarts { + found.push(( + "V5: more restarts than the budget within one window", + format!("instance {inst}: {in_window} > {max_restarts} in {window:?}"), + )); + break; + } + } + continue; + } + // A cohort round is one action however many children it rebuilds, + // charged when it is selected, before the cohort stops and the + // backoff: its rebuilds come later by an unbounded delay, so their + // times are not the grants'. What the factory log does fix: a round + // calls each slot at most once and only while it is current, so the + // next round's grant follows its calls. Split the log where a slot + // repeats: each segment's first call starts a distinct round whose + // grant lies after the previous segment's first call and no later + // than its own first call (a merged segment only undercounts). + // `max_restarts + 1` rounds forced into less than one window break + // V5. (Retries before the tree is ready are per-child grants at + // their call, inside the same bounds.) + let mut firsts: Vec = Vec::new(); + let mut open: Vec<&str> = Vec::new(); + for (t, slot) in log { + if firsts.is_empty() || open.contains(slot) { + firsts.push(*t); + open.clear(); + } + open.push(slot); + } + for i in 0..firsts.len() { + let Some(last) = firsts.get(i + max_restarts) else { + break; + }; + let earliest = if i == 0 { + Duration::ZERO + } else { + firsts[i - 1] + }; + if *last - earliest < window { found.push(( "V5: more restarts than the budget within one window", - format!("instance {inst}: {in_window} > {max_restarts} in {window:?}"), + format!( + "instance {inst}: {} rounds within {:?} < {window:?}", + max_restarts + 1, + *last - earliest + ), )); break; } diff --git a/fuzz/src/oracles.rs b/fuzz/src/oracles.rs index 8729325ca..63ca4d31c 100644 --- a/fuzz/src/oracles.rs +++ b/fuzz/src/oracles.rs @@ -99,6 +99,7 @@ fn upstream(strategy: SupervisorStrategy, j: usize, k: usize) -> bool { SupervisorStrategy::OneForOne => false, SupervisorStrategy::OneForAll => k != j, SupervisorStrategy::RestForOne => k < j, + _ => false, // #[non_exhaustive]: a later strategy is not judged here } } diff --git a/fuzz/src/scenario.rs b/fuzz/src/scenario.rs index 2f011237c..ac35e9fa7 100644 --- a/fuzz/src/scenario.rs +++ b/fuzz/src/scenario.rs @@ -255,7 +255,7 @@ pub enum Op { DropSystem { sys: u8, }, - /// `request_raw`, then drop the reply receiver (R4: not a cancellation). + /// `send_request`, then drop the reply receiver (R4: not a cancellation). DropReply { to: u8, cmd: Cmd, diff --git a/fuzz/src/world.rs b/fuzz/src/world.rs index d4627b6a8..a1ffe0b59 100644 --- a/fuzz/src/world.rs +++ b/fuzz/src/world.rs @@ -84,6 +84,9 @@ pub struct SupInst { pub root: usize, pub max_restarts: usize, pub window: Duration, + /// OneForAll or RestForOne: a recovery round rebuilds several children + /// on one restart grant (0.8, V5). + pub cohort: bool, pub strategy: SupervisorStrategy, pub backoff: Backoff, /// A root whose declared children are all workers (the restart oracles @@ -398,80 +401,90 @@ impl Worker { } impl Handler for Worker { - async fn handle(&mut self, msg: Do, ctx: &Context) -> u64 { - { - let mut s = self.world.lock(); - self.turn(&mut s, "Do"); - if msg.timer != 0 { - s.timers[msg.timer - 1].handled += 1; - } - if msg.nonce != 0 { - if let Some(other) = s.handled.insert(msg.nonce, self.instance) { - let detail = format!("nonce {} by w{other} and w{}", msg.nonce, self.instance); - s.violate("D2: a message was handled twice", detail); + async fn handle( + &mut self, + msg: Do, + __reply: spawned_concurrency::PendingReply<'_, u64>, + ctx: &Context, + ) { + let __value: u64 = async { + { + let mut s = self.world.lock(); + self.turn(&mut s, "Do"); + if msg.timer != 0 { + s.timers[msg.timer - 1].handled += 1; } - if msg.lane == 0 { - let inst = &mut s.insts[self.instance]; - let last = inst.last_lane0; - inst.last_lane0 = msg.nonce; - if msg.nonce <= last { - let detail = format!("w{}: {} after {last}", self.instance, msg.nonce); - s.violate("D1: per-sender FIFO broken", detail); + if msg.nonce != 0 { + if let Some(other) = s.handled.insert(msg.nonce, self.instance) { + let detail = + format!("nonce {} by w{other} and w{}", msg.nonce, self.instance); + s.violate("D2: a message was handled twice", detail); } - } - } - } - match msg.act { - Act::Noop => {} - Act::Panic => { - self.world.lock().insts[self.instance].panicked = true; - panic!("fuzz: injected handler panic"); - } - Act::Sleep(d) => rt::sleep(d).await, - Act::StopSelf(reason) => ctx.stop_with_reason(reason), - Act::Note(to) => { - let _ = to.send(Note { from: ctx.id() }); - } - Act::Link(peer) => { - self.uncertain(ctx.id(), Some(peer.id())); - ctx.link(&peer); - } - Act::Unlink(peer) => { - self.uncertain(ctx.id(), Some(peer.id())); - ctx.unlink(&peer); - } - Act::Monitor(target) => { - let mref = ctx.monitor(&target); - self.world.lock().monitors.push(MonRec { - watcher: ctx.id(), - target, - mref, - demonitored: false, - outside: false, - }); - } - Act::Trap(on) => { - self.uncertain(ctx.id(), None); - ctx.trap_exit(on); - } - Act::Register(name) => { - let _ = ctx.register_name(&name); - } - Act::Helper(v) => { - let _ = ctx.spawn_helper(async move { - match v % 3 { - 0 => std::future::pending::<()>().await, - 1 => rt::sleep(Duration::from_millis(10 * v as u64)).await, - _ => panic!("fuzz: injected helper panic"), + if msg.lane == 0 { + let inst = &mut s.insts[self.instance]; + let last = inst.last_lane0; + inst.last_lane0 = msg.nonce; + if msg.nonce <= last { + let detail = format!("w{}: {} after {last}", self.instance, msg.nonce); + s.violate("D1: per-sender FIFO broken", detail); + } } - }); + } } - Act::Stop(target, reason) => { - self.world.lock().stoppable(target.id()); - target.stop_with_reason(reason); + match msg.act { + Act::Noop => {} + Act::Panic => { + self.world.lock().insts[self.instance].panicked = true; + panic!("fuzz: injected handler panic"); + } + Act::Sleep(d) => rt::sleep(d).await, + Act::StopSelf(reason) => ctx.stop_with_reason(reason), + Act::Note(to) => { + let _ = to.send(Note { from: ctx.id() }); + } + Act::Link(peer) => { + self.uncertain(ctx.id(), Some(peer.id())); + ctx.link(&peer); + } + Act::Unlink(peer) => { + self.uncertain(ctx.id(), Some(peer.id())); + ctx.unlink(&peer); + } + Act::Monitor(target) => { + let mref = ctx.monitor(&target); + self.world.lock().monitors.push(MonRec { + watcher: ctx.id(), + target, + mref, + demonitored: false, + outside: false, + }); + } + Act::Trap(on) => { + self.uncertain(ctx.id(), None); + ctx.trap_exit(on); + } + Act::Register(name) => { + let _ = ctx.register_name(&name); + } + Act::Helper(v) => { + let _ = ctx.spawn_helper(async move { + match v % 3 { + 0 => std::future::pending::<()>().await, + 1 => rt::sleep(Duration::from_millis(10 * v as u64)).await, + _ => panic!("fuzz: injected helper panic"), + } + }); + } + Act::Stop(target, reason) => { + self.world.lock().stoppable(target.id()); + target.stop_with_reason(reason); + } } + msg.nonce } - msg.nonce + .await; + __reply.reply(__value); } } @@ -488,45 +501,63 @@ impl Worker { } impl Handler for Worker { - async fn handle(&mut self, down: Down, ctx: &Context) { - let mut s = self.world.lock(); - self.turn(&mut s, "Down"); - let me = ctx.id(); - let target = s - .monitors - .iter() - .find(|m| m.mref == down.monitor_ref) - .map(|m| (m.watcher, m.target.id())); - match target { - Some((watcher, target)) if watcher == me => { - s.insts[self.instance].ended.insert(target); + async fn handle( + &mut self, + down: Down, + __reply: spawned_concurrency::PendingReply<'_, ()>, + ctx: &Context, + ) { + let __value: () = async { + let mut s = self.world.lock(); + self.turn(&mut s, "Down"); + let me = ctx.id(); + let target = s + .monitors + .iter() + .find(|m| m.mref == down.monitor_ref) + .map(|m| (m.watcher, m.target.id())); + match target { + Some((watcher, target)) if watcher == me => { + s.insts[self.instance].ended.insert(target); + } + Some(_) => s.violate( + "X1: Down delivered to an actor that did not create the monitor", + format!("w{}", self.instance), + ), + None => s.violate( + "X1: Down for an unknown monitor", + format!("w{}", self.instance), + ), + } + let inst = &mut s.insts[self.instance]; + let duplicate = inst.downs.iter().any(|(r, _)| *r == down.monitor_ref); + inst.downs.push((down.monitor_ref, down.reason)); + if duplicate { + s.violate("X1: Down delivered twice", format!("w{}", self.instance)); } - Some(_) => s.violate( - "X1: Down delivered to an actor that did not create the monitor", - format!("w{}", self.instance), - ), - None => s.violate( - "X1: Down for an unknown monitor", - format!("w{}", self.instance), - ), - } - let inst = &mut s.insts[self.instance]; - let duplicate = inst.downs.iter().any(|(r, _)| *r == down.monitor_ref); - inst.downs.push((down.monitor_ref, down.reason)); - if duplicate { - s.violate("X1: Down delivered twice", format!("w{}", self.instance)); } + .await; + __reply.reply(__value); } } impl Handler for Worker { - async fn handle(&mut self, note: Note, _ctx: &Context) { - let mut s = self.world.lock(); - self.turn(&mut s, "Note"); - if s.insts[self.instance].ended.contains(¬e.from) { - let detail = format!("from {} at w{}", s.name(note.from), self.instance); - s.violate("X7: a message arrived after the sender's Down/Exit", detail); + async fn handle( + &mut self, + note: Note, + __reply: spawned_concurrency::PendingReply<'_, ()>, + _ctx: &Context, + ) { + let __value: () = async { + let mut s = self.world.lock(); + self.turn(&mut s, "Note"); + if s.insts[self.instance].ended.contains(¬e.from) { + let detail = format!("from {} at w{}", s.name(note.from), self.instance); + s.violate("X7: a message arrived after the sender's Down/Exit", detail); + } } + .await; + __reply.reply(__value); } } diff --git a/fuzz/tests/cohort_rounds_charged_at_selection.rs b/fuzz/tests/cohort_rounds_charged_at_selection.rs new file mode 100644 index 000000000..403cb1d9f --- /dev/null +++ b/fuzz/tests/cohort_rounds_charged_at_selection.rs @@ -0,0 +1,107 @@ +//! Generated by `cargo run --example repro -- --emit cohort_rounds_charged_at_selection`. +//! +//! Failure: none (passing input) +//! Strategy dfs:bound=0, seed 18446744071398511597. +//! +//! ```text +//! Scenario { +//! seed: 18446744071398511597, +//! sched: Canonical, +//! ops: [ +//! TrapExit { +//! who: 237, +//! on: false, +//! }, +//! NewSystem, +//! Supervisor( +//! SupSpec { +//! system: 10, +//! policy: Policy { +//! strategy: 10, +//! max_restarts: 37, +//! window: 35, +//! backoff: 75, +//! jitter: false, +//! stable: 0, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 81, +//! group: 81, +//! fail_mask: 81, +//! panic_mask: 81, +//! hook_mask: 81, +//! hooks: Hooks { +//! flags: 81, +//! started_sleep: 81, +//! stopped_sleep: 81, +//! }, +//! startup_timeout: 81, +//! shutdown_timeout: 81, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 82, +//! group: 237, +//! fail_mask: 188, +//! panic_mask: 180, +//! hook_mask: 188, +//! hooks: Hooks { +//! flags: 188, +//! started_sleep: 188, +//! stopped_sleep: 188, +//! }, +//! startup_timeout: 188, +//! shutdown_timeout: 25, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Link { +//! a: 0, +//! b: 75, +//! }, +//! Timer { +//! to: 5, +//! ms: 65535, +//! cmd: Monitor( +//! 47, +//! ), +//! interval: true, +//! }, +//! Request { +//! to: 31, +//! cmd: Panic, +//! timeout: None, +//! detached: true, +//! }, +//! Request { +//! to: 31, +//! cmd: Panic, +//! timeout: Some( +//! 0, +//! ), +//! detached: false, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/cohort_rounds_charged_at_selection.input"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn cohort_rounds_charged_at_selection_tape() { + spawned_fuzz::assert_passes(INPUT, None); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn cohort_rounds_charged_at_selection_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} diff --git a/fuzz/tests/late_link_noproc_exit.rs b/fuzz/tests/late_link_noproc_exit.rs new file mode 100644 index 000000000..8e5a75edf --- /dev/null +++ b/fuzz/tests/late_link_noproc_exit.rs @@ -0,0 +1,130 @@ +//! Generated by `cargo run --example repro -- --emit late_link_noproc_exit`. +//! +//! Failure: none (passing input) +//! Strategy random, seed 7466577281530464928. +//! +//! ```text +//! Scenario { +//! seed: 7466577281530464928, +//! sched: Random, +//! ops: [ +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 14, +//! started_sleep: 82, +//! stopped_sleep: 0, +//! }, +//! }, +//! Spawn { +//! how: Managed { +//! group: 38, +//! }, +//! hooks: Hooks { +//! flags: 126, +//! started_sleep: 31, +//! stopped_sleep: 255, +//! }, +//! }, +//! Stop { +//! who: 6, +//! reason: Normal, +//! }, +//! TrapExit { +//! who: 255, +//! on: true, +//! }, +//! Fence, +//! Fence, +//! Fence, +//! Fence, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 247, +//! started_sleep: 30, +//! stopped_sleep: 35, +//! }, +//! }, +//! Monitor { +//! watcher: 1, +//! target: 0, +//! }, +//! DropReply { +//! to: 137, +//! cmd: Note( +//! 122, +//! ), +//! }, +//! Link { +//! a: 103, +//! b: 48, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 66, +//! policy: Policy { +//! strategy: 162, +//! max_restarts: 163, +//! window: 6210, +//! backoff: 14, +//! jitter: false, +//! stable: 0, +//! }, +//! children: [], +//! }, +//! ), +//! Supervisor( +//! SupSpec { +//! system: 152, +//! policy: Policy { +//! strategy: 152, +//! max_restarts: 152, +//! window: 39064, +//! backoff: 152, +//! jitter: false, +//! stable: 152, +//! }, +//! children: [], +//! }, +//! ), +//! Fence, +//! Fence, +//! Fence, +//! Fence, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 31, +//! started_sleep: 136, +//! stopped_sleep: 255, +//! }, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 0, +//! started_sleep: 0, +//! stopped_sleep: 0, +//! }, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/late_link_noproc_exit.input"); +const TAPE: &str = include_str!("../regressions/late_link_noproc_exit.tape"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn late_link_noproc_exit_tape() { + spawned_fuzz::assert_passes(INPUT, Some(TAPE)); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn late_link_noproc_exit_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} diff --git a/fuzz/tests/link_from_an_exiting_actor.rs b/fuzz/tests/link_from_an_exiting_actor.rs new file mode 100644 index 000000000..f25fda826 --- /dev/null +++ b/fuzz/tests/link_from_an_exiting_actor.rs @@ -0,0 +1,118 @@ +//! Generated by `cargo run --example repro -- --emit link_from_an_exiting_actor`. +//! +//! Failure: none (passing input) +//! Strategy random, seed 2531906049332685619. +//! +//! ```text +//! Scenario { +//! seed: 2531906049332685619, +//! sched: Random, +//! ops: [ +//! Link { +//! a: 35, +//! b: 35, +//! }, +//! Link { +//! a: 35, +//! b: 35, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Panic, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 35, +//! started_sleep: 35, +//! stopped_sleep: 35, +//! }, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Normal, +//! ), +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 10, +//! started_sleep: 10, +//! stopped_sleep: 10, +//! }, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 10, +//! started_sleep: 10, +//! stopped_sleep: 10, +//! }, +//! }, +//! Spawn { +//! how: Managed { +//! group: 170, +//! }, +//! hooks: Hooks { +//! flags: 170, +//! started_sleep: 95, +//! stopped_sleep: 95, +//! }, +//! }, +//! Stop { +//! who: 95, +//! reason: Kill, +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Kill, +//! ), +//! }, +//! CrashChild { +//! sup: 170, +//! child: 170, +//! how: Stop( +//! Shutdown, +//! ), +//! }, +//! Link { +//! a: 35, +//! b: 154, +//! }, +//! TrapExit { +//! who: 1, +//! on: false, +//! }, +//! Spawn { +//! how: Unmanaged, +//! hooks: Hooks { +//! flags: 0, +//! started_sleep: 0, +//! stopped_sleep: 0, +//! }, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/link_from_an_exiting_actor.input"); +const TAPE: &str = include_str!("../regressions/link_from_an_exiting_actor.tape"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn link_from_an_exiting_actor_tape() { + spawned_fuzz::assert_passes(INPUT, Some(TAPE)); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn link_from_an_exiting_actor_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} diff --git a/fuzz/tests/panic_then_late_shutdown_notice.rs b/fuzz/tests/panic_then_late_shutdown_notice.rs new file mode 100644 index 000000000..83add4e9f --- /dev/null +++ b/fuzz/tests/panic_then_late_shutdown_notice.rs @@ -0,0 +1,425 @@ +//! Generated by `cargo run --example repro -- --emit panic_then_late_shutdown_notice`. +//! +//! Failure: none (passing input) +//! Strategy dfs:bound=0, seed 14490781309383357083. +//! +//! ```text +//! Scenario { +//! seed: 14490781309383357083, +//! sched: Canonical, +//! ops: [ +//! CrashChild { +//! sup: 204, +//! child: 45, +//! how: Panic, +//! }, +//! Advance { +//! ms: 1002, +//! }, +//! CrashChild { +//! sup: 51, +//! child: 131, +//! how: Stop( +//! Kill, +//! ), +//! }, +//! StopSupervisor { +//! sup: 170, +//! mode: 111, +//! ms: 7477, +//! }, +//! DropReply { +//! to: 136, +//! cmd: Sleep( +//! 36, +//! ), +//! }, +//! CrashChild { +//! sup: 234, +//! child: 109, +//! how: Stop( +//! Normal, +//! ), +//! }, +//! Supervisor( +//! SupSpec { +//! system: 148, +//! policy: Policy { +//! strategy: 181, +//! max_restarts: 0, +//! window: 3540, +//! backoff: 239, +//! jitter: false, +//! stable: 89, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 210, +//! group: 69, +//! fail_mask: 247, +//! panic_mask: 186, +//! hook_mask: 229, +//! hooks: Hooks { +//! flags: 6, +//! started_sleep: 44, +//! stopped_sleep: 152, +//! }, +//! startup_timeout: 213, +//! shutdown_timeout: 78, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Unlink { +//! a: 232, +//! b: 147, +//! }, +//! Fence, +//! TrapExit { +//! who: 39, +//! on: true, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 143, +//! policy: Policy { +//! strategy: 174, +//! max_restarts: 131, +//! window: 40554, +//! backoff: 239, +//! jitter: false, +//! stable: 165, +//! }, +//! children: [], +//! }, +//! ), +//! DropSystem { +//! sys: 217, +//! }, +//! Request { +//! to: 91, +//! cmd: Note( +//! 157, +//! ), +//! timeout: None, +//! detached: false, +//! }, +//! WaitGeneration { +//! sup: 229, +//! child: 178, +//! ms: 53067, +//! }, +//! Request { +//! to: 159, +//! cmd: Note( +//! 20, +//! ), +//! timeout: None, +//! detached: false, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 36, +//! }, +//! hooks: Hooks { +//! flags: 166, +//! started_sleep: 153, +//! stopped_sleep: 210, +//! }, +//! }, +//! WaitGeneration { +//! sup: 199, +//! child: 26, +//! ms: 22094, +//! }, +//! CrashChild { +//! sup: 121, +//! child: 165, +//! how: Panic, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 70, +//! policy: Policy { +//! strategy: 100, +//! max_restarts: 10, +//! window: 49640, +//! backoff: 124, +//! jitter: false, +//! stable: 70, +//! }, +//! children: [], +//! }, +//! ), +//! Demonitor { +//! which: 255, +//! }, +//! DropSystem { +//! sys: 83, +//! }, +//! StopSupervisor { +//! sup: 252, +//! mode: 33, +//! ms: 39673, +//! }, +//! NewSystem, +//! NewSystem, +//! WaitGeneration { +//! sup: 51, +//! child: 229, +//! ms: 7648, +//! }, +//! Unlink { +//! a: 218, +//! b: 31, +//! }, +//! CancelTimer { +//! which: 165, +//! }, +//! Request { +//! to: 220, +//! cmd: Register( +//! 6, +//! ), +//! timeout: None, +//! detached: true, +//! }, +//! NewSystem, +//! NewSystem, +//! DropSystem { +//! sys: 10, +//! }, +//! CrashChild { +//! sup: 112, +//! child: 36, +//! how: Panic, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 196, +//! }, +//! hooks: Hooks { +//! flags: 5, +//! started_sleep: 27, +//! stopped_sleep: 95, +//! }, +//! }, +//! Request { +//! to: 55, +//! cmd: Link( +//! 71, +//! ), +//! timeout: Some( +//! 36926, +//! ), +//! detached: true, +//! }, +//! Spawn { +//! how: Linked { +//! parent: 123, +//! }, +//! hooks: Hooks { +//! flags: 163, +//! started_sleep: 44, +//! stopped_sleep: 50, +//! }, +//! }, +//! NewSystem, +//! Stop { +//! who: 22, +//! reason: Shutdown, +//! }, +//! Stop { +//! who: 100, +//! reason: Normal, +//! }, +//! Link { +//! a: 100, +//! b: 188, +//! }, +//! Request { +//! to: 108, +//! cmd: Panic, +//! timeout: Some( +//! 55704, +//! ), +//! detached: false, +//! }, +//! DropReply { +//! to: 84, +//! cmd: Monitor( +//! 245, +//! ), +//! }, +//! Fence, +//! Demonitor { +//! which: 66, +//! }, +//! Supervisor( +//! SupSpec { +//! system: 32, +//! policy: Policy { +//! strategy: 90, +//! max_restarts: 146, +//! window: 29829, +//! backoff: 72, +//! jitter: false, +//! stable: 178, +//! }, +//! children: [ +//! Worker( +//! WorkerDef { +//! restart: 216, +//! group: 149, +//! fail_mask: 158, +//! panic_mask: 255, +//! hook_mask: 240, +//! hooks: Hooks { +//! flags: 161, +//! started_sleep: 204, +//! stopped_sleep: 205, +//! }, +//! startup_timeout: 36, +//! shutdown_timeout: 250, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 22, +//! group: 228, +//! fail_mask: 185, +//! panic_mask: 20, +//! hook_mask: 252, +//! hooks: Hooks { +//! flags: 217, +//! started_sleep: 218, +//! stopped_sleep: 135, +//! }, +//! startup_timeout: 128, +//! shutdown_timeout: 99, +//! }, +//! ), +//! Worker( +//! WorkerDef { +//! restart: 104, +//! group: 220, +//! fail_mask: 233, +//! panic_mask: 162, +//! hook_mask: 41, +//! hooks: Hooks { +//! flags: 173, +//! started_sleep: 24, +//! stopped_sleep: 184, +//! }, +//! startup_timeout: 163, +//! shutdown_timeout: 34, +//! }, +//! ), +//! ], +//! }, +//! ), +//! Yield { +//! n: 186, +//! }, +//! Demonitor { +//! which: 177, +//! }, +//! Monitor { +//! watcher: 91, +//! target: 175, +//! }, +//! Fence, +//! Advance { +//! ms: 13884, +//! }, +//! NewSystem, +//! CancelTimer { +//! which: 86, +//! }, +//! NewSystem, +//! Yield { +//! n: 126, +//! }, +//! StopSupervisor { +//! sup: 148, +//! mode: 79, +//! ms: 29051, +//! }, +//! Send { +//! to: 76, +//! cmd: Noop, +//! via_recipient: false, +//! }, +//! Fence, +//! Advance { +//! ms: 33906, +//! }, +//! Monitor { +//! watcher: 216, +//! target: 27, +//! }, +//! StopSupervisor { +//! sup: 190, +//! mode: 135, +//! ms: 44670, +//! }, +//! Yield { +//! n: 50, +//! }, +//! Request { +//! to: 89, +//! cmd: Stop( +//! 227, +//! Shutdown, +//! ), +//! timeout: Some( +//! 48379, +//! ), +//! detached: true, +//! }, +//! Request { +//! to: 240, +//! cmd: StopSelf( +//! Panic, +//! ), +//! timeout: Some( +//! 29471, +//! ), +//! detached: true, +//! }, +//! Spawn { +//! how: Managed { +//! group: 80, +//! }, +//! hooks: Hooks { +//! flags: 224, +//! started_sleep: 141, +//! stopped_sleep: 52, +//! }, +//! }, +//! CrashChild { +//! sup: 81, +//! child: 239, +//! how: StopNested, +//! }, +//! ], +//! } +//! ``` +#![cfg(spawned_sim)] + +const INPUT: &[u8] = include_bytes!("../regressions/panic_then_late_shutdown_notice.input"); + +/// The input under its own schedule and under its recorded tape. +#[test] +fn panic_then_late_shutdown_notice_tape() { + spawned_fuzz::assert_passes(INPUT, None); +} + +/// The same operations under other simulator schedules. +#[spawned_macros::sim_test(seeds = 32)] +async fn panic_then_late_shutdown_notice_seeds(seed: u64) { + spawned_fuzz::init(); + spawned_fuzz::run_body(spawned_fuzz::decode(INPUT).unwrap(), seed).await; +} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 03de9d087..0da382175 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -489,7 +489,9 @@ fn generate_target_impl( let body = match &m.kind { MethodKind::Send => quote! { target.send(message) }, MethodKind::Request(_) => match mode { - RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from_with_timeout(target.request_raw(message), timeout) }, + // The timeout was chosen before the message was built; the + // deadline is fixed now, at send, and travels in the slot. + RuntimeMode::Tasks => quote! { spawned_concurrency::Response::from(target.send_request(message, spawned_concurrency::Deadline::after(timeout))) }, RuntimeMode::Threads => quote! { spawned_concurrency::Response::ready(target.request_with_timeout(message, timeout)) }, }, }; @@ -722,7 +724,9 @@ struct ProtocolMethodInfo { /// supports conversion with the generated `ToXRef` trait. Configure before /// erasing the concrete handle. These are reply-wait limits, not cancellation /// or inherited deadlines; accepted work may still execute after timeout. -/// Native protocol calls block immediately; async response timers start on poll. +/// The deadline is fixed when the request is sent (send time plus the +/// timeout), as in OTP's `gen_server:call`. Native protocol calls block +/// immediately; async responses wait until that deadline when polled. #[proc_macro_attribute] pub fn protocol(attr: TokenStream, item: TokenStream) -> TokenStream { let options = parse_macro_input!(attr with syn::punctuated::Punctuated::::parse_terminated); @@ -1073,7 +1077,10 @@ pub fn protocol(attr: TokenStream, item: TokenStream) -> TokenStream { fn is_handler_attribute(attr: &Attribute) -> bool { let path = attr.path(); - path.is_ident("handler") || path.is_ident("send_handler") || path.is_ident("request_handler") + path.is_ident("handler") + || path.is_ident("send_handler") + || path.is_ident("request_handler") + || path.is_ident("deferred_handler") } fn is_actor_method_attribute(attr: &Attribute) -> bool { @@ -1109,6 +1116,20 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// impl ChatUser { /* ... */ } /// ``` /// +/// # Deferred-reply bounds +/// +/// `max_outstanding = N` sets `Actor::MAX_OUTSTANDING` (deferred credits per +/// instance) and `max_native_jobs = N` sets `Actor::MAX_NATIVE_JOBS` (blocking +/// jobs running at once; the default, `min(4, available_parallelism)`, is +/// easy to outgrow: under load the only symptom is `NativeBudget` refusals). +/// Both take any constant expression and combine with `protocol`; a +/// supervised child can still override them per `ChildSpec`: +/// +/// ```ignore +/// #[actor(protocol = P2PProtocol, max_native_jobs = 64, max_outstanding = 4096)] +/// impl P2PServer { /* ... */ } +/// ``` +/// /// # Lints /// /// `#[actor]` warns (compile time only) when the impl block builds or accepts @@ -1122,12 +1143,16 @@ fn is_actor_method_attribute(attr: &Attribute) -> bool { /// |-----------|---------| /// | `#[request_handler]` | Messages that expect a reply (`Response`) | /// | `#[send_handler]` | Fire-and-forget messages | -/// | `#[handler]` | Generic handler (works for either kind) | +/// | `#[deferred_handler]` | Requests answered through a `PendingReply` (now, or later via `defer()`) | /// /// Handler signature: `fn name(&mut self, msg: MessageType, ctx: &Context) -> ReturnType` /// /// The return type must match the message's `Message::Result`. For request handlers -/// returning `()`, omit the return type. +/// returning `()`, omit the return type. The generated `Handler::handle` calls +/// the method and replies with its result; the method stays callable directly. +/// +/// A deferred handler takes the reply obligation instead of returning: +/// `fn name(&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context)`. /// /// # Lifecycle Hooks /// @@ -1211,7 +1236,10 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { let (impl_generics, _, where_clause) = impl_block.generics.split_for_impl(); // --- Parse named parameters: #[actor(protocol = X)], #[actor(protocol(X, Y))], + // the deferred-reply bounds `max_outstanding = N`, `max_native_jobs = N`, // and the lint opt-outs `allow_shared_state` and `allow_panic_abort` --- + let mut max_outstanding: Option = None; + let mut max_native_jobs: Option = None; let mut allow_shared_state = false; let mut allow_panic_abort = false; let bridge_traits: Vec = if attr.is_empty() { @@ -1233,15 +1261,26 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { let mut protocols = Vec::new(); while !input.is_empty() { let key: Ident = input.parse()?; - if key == "allow_shared_state" { + if key == "max_outstanding" || key == "max_native_jobs" { + let _: syn::Token![=] = input.parse()?; + let value: syn::Expr = input.parse()?; + let slot = if key == "max_outstanding" { + &mut max_outstanding + } else { + &mut max_native_jobs + }; + if slot.replace(value).is_some() { + return Err(syn::Error::new(key.span(), format!("`{key}` given twice"))); + } + } else if key == "allow_shared_state" { allow_shared_state = true; } else if key == "allow_panic_abort" { allow_panic_abort = true; } else if key != "protocol" { return Err(syn::Error::new( key.span(), - "unknown parameter, expected `protocol`, `allow_shared_state` \ - or `allow_panic_abort`", + "unknown parameter, expected `protocol`, `max_outstanding`, \ + `max_native_jobs`, `allow_shared_state` or `allow_panic_abort`", )); } else if input.peek(syn::Token![=]) { // protocol = TraitName @@ -1301,14 +1340,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if is_started - || is_stopped - || method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) - { + if is_started || is_stopped || method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned(&method.sig, "#[exit_received] cannot be combined with other lifecycle or handler attributes").to_compile_error().into(); } if method.sig.inputs.len() != 3 @@ -1350,11 +1382,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) { + if method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned( &method.sig, "#[started] cannot be combined with handler attributes", @@ -1400,11 +1428,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - if method.attrs.iter().any(|a| { - a.path().is_ident("handler") - || a.path().is_ident("send_handler") - || a.path().is_ident("request_handler") - }) { + if method.attrs.iter().any(is_handler_attribute) { return syn::Error::new_spanned( &method.sig, "#[stopped] cannot be combined with handler attributes", @@ -1450,18 +1474,24 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { for item in &mut impl_block.items { if let ImplItem::Fn(method) = item { - let handler_idx = method.attrs.iter().position(|attr| { - attr.path().is_ident("handler") - || attr.path().is_ident("send_handler") - || attr.path().is_ident("request_handler") - }); + let handler_idx = method.attrs.iter().position(is_handler_attribute); if let Some(idx) = handler_idx { + let deferred = method.attrs[idx].path().is_ident("deferred_handler"); + let send = method.attrs[idx].path().is_ident("send_handler"); + if method.attrs[idx].path().is_ident("handler") { + return syn::Error::new_spanned( + &method.attrs[idx], + "#[handler] was removed in 0.8: use #[send_handler] (returns `()`, no reply) or #[request_handler] (answers with the return value)", + ) + .to_compile_error() + .into(); + } method.attrs.remove(idx); if let Some(extra) = method.attrs.iter().find(|a| is_handler_attribute(a)) { return syn::Error::new_spanned( extra, - "a method can carry only one of #[handler], #[send_handler] and #[request_handler]", + "a method can carry only one of #[send_handler], #[request_handler] and #[deferred_handler]", ) .to_compile_error() .into(); @@ -1481,9 +1511,7 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .attrs .iter() .filter(|a| { - !a.path().is_ident("handler") - && !a.path().is_ident("send_handler") - && !a.path().is_ident("request_handler") + !is_handler_attribute(a) && !a.path().is_ident("started") && !a.path().is_ident("stopped") }) @@ -1495,9 +1523,21 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { has_async = true; } - // Validate handler has exactly 3 parameters: &mut self, msg, ctx + // Validate parameters: (&mut self, msg, ctx), or for a deferred + // handler (&mut self, msg, reply, ctx). let param_count = method.sig.inputs.len(); - if param_count != 3 { + let ctx_index = if deferred { 3 } else { 2 }; + if deferred && param_count != 4 { + return syn::Error::new_spanned( + &method.sig, + format!( + "#[deferred_handler] method must have 4 parameters (&mut self, msg: M, reply: PendingReply<'_, R>, ctx: &Context), found {param_count}" + ), + ) + .to_compile_error() + .into(); + } + if !deferred && param_count != 3 { return syn::Error::new_spanned( &method.sig, format!( @@ -1507,6 +1547,33 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } + if deferred { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned( + ty, + "#[deferred_handler] answers through its PendingReply; remove the return type", + ) + .to_compile_error() + .into(); + } + } + } + // A send handler that returned a value would silently answer a + // `request` with it. (`impl Future` returns are checked by the + // `Result = ()` bound of the message instead.) + if send { + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) && !matches!(**ty, Type::ImplTrait(_)) { + return syn::Error::new_spanned( + ty, + "#[send_handler] methods return `()`; use #[request_handler] to answer a request with a value", + ) + .to_compile_error() + .into(); + } + } + } // `&self` is accepted too; a missing or by-value receiver is not. match method.sig.inputs.first() { @@ -1520,9 +1587,9 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { .to_compile_error() .into(); } - None => unreachable!("three parameters were checked above"), + None => unreachable!("the parameters were checked above"), } - if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(2) { + if let Some(FnArg::Typed(ctx)) = method.sig.inputs.iter().nth(ctx_index) { if !matches!(&*ctx.ty, Type::Reference(r) if r.mutability.is_none()) { return syn::Error::new_spanned( &ctx.ty, @@ -1546,29 +1613,55 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { } }; - // Keep an omitted return type omitted: rustc then suggests - // adding `-> R` after the user's signature for a request reply. - let output = match &method.sig.output { - ReturnType::Default => quote! {}, - ReturnType::Type(_, ty) => quote! { -> #ty }, + // The generated `handle` is the envelope dispatch: it takes the + // request's reply obligation and answers it with the method's + // result (a deferred handler receives the obligation itself). + let reply_ty = quote! { + spawned_concurrency::PendingReply<'_, <#msg_ty as spawned_concurrency::message::Message>::Result> }; - // The parameter list carries the user's parentheses span, so - // that suggestion lands after the user's `)`. + // The parameter list carries the user's parentheses span. let mut params = proc_macro2::Group::new( proc_macro2::Delimiter::Parenthesis, - quote! { &mut self, msg: #msg_ty, ctx: &Context }, + quote! { &mut self, msg: #msg_ty, reply: #reply_ty, ctx: &Context }, ); params.set_span(method.sig.paren_token.span.join()); // Span the generated impl at the method so trait errors // (conflicting impls, signature mismatches) point at it. let span = method.sig.span(); + // The reply's type must equal the message's `Result`; a + // mismatch is reported at the user's return type. + let ret_span = match &method.sig.output { + ReturnType::Default => method.sig.paren_token.span.join(), + ReturnType::Type(_, ty) => ty.span(), + }; + // A tasks handler may also be a plain `fn` returning + // `impl Future`; its result is awaited like an `async fn`'s. + let returns_future = matches!( + &method.sig.output, + ReturnType::Type(_, ty) if matches!(**ty, Type::ImplTrait(_)) + ); + let asynchronous = method.sig.asyncness.is_some() || returns_future; + let call = match (deferred, asynchronous) { + (true, true) => quote! { self.#method_name(msg, reply, ctx).await }, + (true, false) => quote! { self.#method_name(msg, reply, ctx) }, + (false, true) => { + let value = + quote::quote_spanned! { ret_span => self.#method_name(msg, ctx).await }; + quote! { let _ = reply.reply(#value); } + } + (false, false) => { + let value = + quote::quote_spanned! { ret_span => self.#method_name(msg, ctx) }; + quote! { let _ = reply.reply(#value); } + } + }; - let handler_impl = if method.sig.asyncness.is_some() { + let handler_impl = if asynchronous { quote::quote_spanned! { span => #(#extra_attrs)* impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - async fn handle #params #output { - self.#method_name(msg, ctx).await + async fn handle #params { + #call } } } @@ -1576,8 +1669,8 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { quote::quote_spanned! { span => #(#extra_attrs)* impl #impl_generics Handler<#msg_ty> for #self_ty #where_clause { - fn handle #params #output { - self.#method_name(msg, ctx) + fn handle #params { + #call } } } @@ -1613,9 +1706,17 @@ pub fn actor(attr: TokenStream, item: TokenStream) -> TokenStream { quote! { #[doc = #doc_body] } }; + let bound_consts = { + let outstanding = max_outstanding.map(|n| quote! { const MAX_OUTSTANDING: usize = #n; }); + let native = max_native_jobs + .map(|n| quote! { const MAX_NATIVE_JOBS: ::core::option::Option = ::core::option::Option::Some(#n); }); + quote! { #outstanding #native } + }; + let actor_impl = quote! { #protocol_doc impl #impl_generics Actor for #self_ty #where_clause { + #bound_consts #(#lifecycle_methods)* } }; @@ -1812,6 +1913,133 @@ fn actor_lints( // --- #[sim_test] --- +/// Send-only form of a manual `Handler` impl: write `handle` without the +/// reply parameter, and the impl answers `()` once `handle` returns. +/// +/// ```ignore +/// #[send_handler] +/// impl Handler for Poller { +/// async fn handle(&mut self, _msg: Tick, ctx: &Context) { +/// self.poll().await; +/// } +/// } +/// ``` +/// +/// expands to the full `handle(&mut self, msg, reply: PendingReply<'_, ()>, +/// ctx)` whose body runs yours and then calls `reply.reply(())`, also after +/// an early `return`. The automatic `()` is a real reply, as with +/// `#[send_handler]` methods in an `#[actor]` block: a `request` gets +/// `Ok(())`, not `ReplyDropped`, and a panic in the body answers +/// `ActorStopped` as for any handler. The message's `Result` must be `()`. +/// Works for `tasks` (`async fn`) and `threads` (`fn`) handlers. +/// +/// The reply type is named through the `Handler` path when it is qualified +/// (`impl my_spawned::tasks::Handler for A` uses +/// `my_spawned::tasks::PendingReply`), so a renamed dependency works. +/// With a bare `Handler`, it is `spawned_concurrency::PendingReply`; +/// a crate that renames `spawned-concurrency` names it: +/// `#[send_handler(crate = my_spawned)]`. +#[proc_macro_attribute] +pub fn send_handler(attr: TokenStream, item: TokenStream) -> TokenStream { + let krate: Option = if attr.is_empty() { + None + } else { + let parser = |input: syn::parse::ParseStream| { + input.parse::()?; + input.parse::()?; + let path: syn::Path = input.parse()?; + if !input.is_empty() { + return Err(input.error("expected only `crate = path`")); + } + Ok(path) + }; + match syn::parse::Parser::parse(parser, attr) { + Ok(path) => Some(path), + Err(error) => { + return syn::Error::new( + error.span(), + format!("#[send_handler] takes no arguments or `crate = path`: {error}"), + ) + .to_compile_error() + .into() + } + } + }; + let mut impl_block = parse_macro_input!(item as ItemImpl); + let is_handler = impl_block + .trait_ + .as_ref() + .and_then(|(_, path, _)| path.segments.last()) + .is_some_and(|segment| segment.ident == "Handler"); + if !is_handler { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] on an item applies to an `impl Handler for Actor` block; \ + inside an #[actor] block, put it on the handler method instead", + ) + .to_compile_error() + .into(); + } + let Some(method) = impl_block.items.iter_mut().find_map(|item| match item { + ImplItem::Fn(method) if method.sig.ident == "handle" => Some(method), + _ => None, + }) else { + return syn::Error::new_spanned( + &impl_block.self_ty, + "#[send_handler] impl must define `handle`", + ) + .to_compile_error() + .into(); + }; + if method.sig.inputs.len() != 3 { + return syn::Error::new_spanned( + &method.sig, + "#[send_handler] `handle` takes (&mut self, msg: M, ctx: &Context), without the reply parameter", + ) + .to_compile_error() + .into(); + } + if let ReturnType::Type(_, ty) = &method.sig.output { + if !is_unit_type(ty) { + return syn::Error::new_spanned(ty, "#[send_handler] `handle` returns nothing") + .to_compile_error() + .into(); + } + } + let reply = format_ident!("__spawned_reply"); + // The `Handler` path's module (`tasks`/`threads`) re-exports + // `PendingReply`; a bare `Handler` falls back to the crate path. + let handler_module = impl_block.trait_.as_ref().and_then(|(_, path, _)| { + (path.segments.len() > 1).then(|| { + let mut module = path.clone(); + module.segments.pop(); + module.segments.pop_punct(); + module + }) + }); + let reply_ty = match (krate, handler_module) { + (Some(krate), _) => quote! { #krate::PendingReply<'_, ()> }, + (None, Some(module)) => quote! { #module::PendingReply<'_, ()> }, + (None, None) => quote! { spawned_concurrency::PendingReply<'_, ()> }, + }; + let reply_param: FnArg = syn::parse_quote! { + #reply: #reply_ty + }; + method.sig.inputs.insert(2, reply_param); + method.sig.output = ReturnType::Default; + let body = &method.block; + let run = if method.sig.asyncness.is_some() { + quote! { async move #body.await } + } else { + quote! { (move || #body)() } + }; + method.block = syn::parse_quote! {{ + let () = #run; + let _ = #reply.reply(()); + }}; + quote! { #impl_block }.into() +} + /// Seeded async test: on Tokio in normal builds, over a seed range on the /// deterministic simulator under `--cfg spawned_sim` (see /// `spawned_rt::tasks::SimTest`). Accepts `#[tokio::test]`'s `flavor` and diff --git a/rt/src/tasks/mod.rs b/rt/src/tasks/mod.rs index 5094e1018..6f3fb8998 100644 --- a/rt/src/tasks/mod.rs +++ b/rt/src/tasks/mod.rs @@ -39,13 +39,11 @@ pub mod affinity; #[cfg(spawned_sim)] pub use crate::tasks::sim::{ block_in_place, sleep, spawn, spawn_blocking, task_id, timeout, unconstrained, yield_now, - Handle, JoinHandle, Runtime, RuntimeFlavor, + Handle, JoinHandle, Runtime, RuntimeFlavor, Sleep, }; pub use crate::tasks::tokio::mpsc; pub use crate::tasks::tokio::oneshot; #[cfg(not(spawned_sim))] -pub use crate::tasks::tokio::sleep; -#[cfg(not(spawned_sim))] pub use crate::tasks::tokio::timeout; pub use crate::tasks::tokio::watch; pub use crate::tasks::tokio::CancellationToken; @@ -54,6 +52,8 @@ pub use crate::tasks::tokio::{ block_in_place, spawn, spawn_blocking, task_id, unconstrained, yield_now, Handle, JoinHandle, Runtime, RuntimeFlavor, }; +#[cfg(not(spawned_sim))] +pub use crate::tasks::tokio::{sleep, Sleep}; pub use crate::tasks::tokio::{BroadcastStream, ReceiverStream}; // Timers and task sets (determinism D3): virtual under the simulator. #[cfg(spawned_sim)] diff --git a/rt/src/tasks/sim.rs b/rt/src/tasks/sim.rs index 14a0c090b..16ec94cf9 100644 --- a/rt/src/tasks/sim.rs +++ b/rt/src/tasks/sim.rs @@ -2376,6 +2376,13 @@ fn running_world() -> Option> { pub struct Instant(Duration); impl Instant { + /// Virtual time since the simulation started (the clock-tick seam). + pub(crate) fn since_start(&self) -> Duration { + self.0 + } + pub(crate) fn from_start(elapsed: Duration) -> Self { + Self(elapsed) + } pub fn now() -> Self { match running_world() { Some(shared) => Self(lock(&shared.state).now), diff --git a/rt/src/tasks/tokio/mod.rs b/rt/src/tasks/tokio/mod.rs index 2f12c0e2b..c9c1e7eea 100644 --- a/rt/src/tasks/tokio/mod.rs +++ b/rt/src/tasks/tokio/mod.rs @@ -10,7 +10,7 @@ pub use tokio::{ task::{ block_in_place, id as task_id, spawn, spawn_blocking, unconstrained, yield_now, JoinHandle, }, - time::{sleep, timeout}, + time::{sleep, timeout, Sleep}, }; pub use tokio_stream::wrappers::{BroadcastStream, UnboundedReceiverStream as ReceiverStream}; pub use tokio_util::sync::CancellationToken; diff --git a/rt/src/time.rs b/rt/src/time.rs index 47b695007..390d2c4d5 100644 --- a/rt/src/time.rs +++ b/rt/src/time.rs @@ -31,3 +31,74 @@ pub fn entropy() -> u64 { ^ NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed) } } + +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise +/// Largest tick value an instant maps to; `u64::MAX` is reserved for "never". +pub const MAX_TICKS: u64 = u64::MAX - 1; + +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise +/// Nanosecond clock ticks of `instant`, for packing deadlines into atomics. +/// +/// Ticks count from a process-wide epoch (the first call), or, under +/// simulation, from the start of the simulated world. Instants before the +/// epoch map to 0 and instants beyond the range saturate at [`MAX_TICKS`]. +pub fn to_ticks(instant: Instant) -> u64 { + let nanos = elapsed_since_epoch(instant).as_nanos(); + nanos.min(MAX_TICKS as u128) as u64 +} + +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise +/// Inverse of [`to_ticks`]. +pub fn from_ticks(ticks: u64) -> Instant { + let elapsed = std::time::Duration::from_nanos(ticks.min(MAX_TICKS)); + #[cfg(all(spawned_sim, feature = "async"))] + { + crate::tasks::sim::Instant::from_start(elapsed) + } + #[cfg(not(all(spawned_sim, feature = "async")))] + { + let epoch = epoch(); + epoch.checked_add(elapsed).unwrap_or(epoch) + } +} + +#[doc(hidden)] // clock plumbing for spawned-concurrency; no stability promise +/// Ticks of [`Instant::now`]. +pub fn now_ticks() -> u64 { + to_ticks(Instant::now()) +} + +#[cfg(not(all(spawned_sim, feature = "async")))] +fn epoch() -> Instant { + static EPOCH: std::sync::OnceLock = std::sync::OnceLock::new(); + *EPOCH.get_or_init(Instant::now) +} + +fn elapsed_since_epoch(instant: Instant) -> std::time::Duration { + #[cfg(all(spawned_sim, feature = "async"))] + { + instant.since_start() + } + #[cfg(not(all(spawned_sim, feature = "async")))] + { + instant.saturating_duration_since(epoch()) + } +} + +#[cfg(test)] +mod tick_tests { + use super::*; + use std::time::Duration; + + #[test] + fn ticks_round_trip_and_order() { + let _ = now_ticks(); // initialize the epoch before `a` + let a = Instant::now(); + let b = a + Duration::from_millis(5); + assert!(to_ticks(b) > to_ticks(a)); + assert_eq!(to_ticks(b) - to_ticks(a), 5_000_000); + let back = from_ticks(to_ticks(b)); + assert_eq!(to_ticks(back), to_ticks(b)); + assert!(now_ticks() >= to_ticks(a)); + } +} diff --git a/scripts/check-api.py b/scripts/check-api.py index 0ffa57935..5fca389ad 100644 --- a/scripts/check-api.py +++ b/scripts/check-api.py @@ -1,5 +1,14 @@ #!/usr/bin/env python3 -"""Check feature builds, migrated APIs and declared v0.7 feature-split breaks.""" +"""Check feature builds, migrated APIs and the release's declared API breaks. + +`verification/api/release.json` names the baseline release and the release +type of the work in progress. A `patch` release must keep the baseline's API +(v0.7's feature-split breaks against v0.6.0 are declared item by item in +`v0.6.0-no-default.json`); a breaking pre-1.0 release (`minor`, as 0.8 is +against 0.7.3, declared `major` to the tool) only checks that both APIs build +in every profile: the tool skips its breaking-change lints for a major +release. +""" import argparse import json import os @@ -45,10 +54,15 @@ def declared_break(command, expected, root): def semver(root, baseline): + release = json.loads((root / "verification/api/release.json").read_text()) + baseline = baseline or release["baseline"] + release_type = release["release_type"] if baseline == release["baseline"] else "patch" + if release_type != "patch": + print(f"Breaking release against {baseline} ({release_type}): {release['reason']}", flush=True) declarations = json.loads((root / "verification/api/v0.6.0-no-default.json").read_text()) - if baseline != declarations["baseline"]: + if release_type == "patch" and baseline != declarations["baseline"]: raise ValueError("the declared feature-split audit requires baseline " + declarations["baseline"]) - base = ["cargo", "semver-checks", "--color", "never", "--release-type", "patch", "--baseline-rev", baseline] + base = ["cargo", "semver-checks", "--color", "never", "--release-type", release_type, "--baseline-rev", baseline] libraries = ["-p", "spawned-concurrency", "-p", "spawned-rt"] # Old no-default consumers restore the previous API by explicitly enabling # async. Check this migration as well as defaults, with/without metrics. @@ -60,6 +74,10 @@ def semver(root, baseline): ]: print("Checking API profile: " + profile, flush=True) run(*base, *features, *packages, cwd=root) + if release_type != "patch": + # Breaks are allowed and listed in MIGRATION; the item-level audit + # applies to the v0.6.0 -> v0.7 patch line only. + return for package, expected in declarations["packages"].items(): print("Auditing raw no-default API: " + package, flush=True) command = base + ["--only-explicit-features", "-p", package] @@ -104,7 +122,8 @@ def features(root, cache, offline): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("check", choices=("features", "semver")) - parser.add_argument("--baseline", default="v0.6.0") + # Default: the baseline of verification/api/release.json. + parser.add_argument("--baseline", default=None) parser.add_argument("--offline", action="store_true") args = parser.parse_args() if args.offline: diff --git a/testing/src/faults.rs b/testing/src/faults.rs index 4ff2629a5..d94ff025c 100644 --- a/testing/src/faults.rs +++ b/testing/src/faults.rs @@ -1,111 +1,34 @@ //! Fault injection through public lifecycle APIs. //! -//! [`exit`] makes an actor exit with a chosen [`ExitReason`]. It resembles -//! Erlang's `exit(Pid, Reason)` for `Panic`, delivered through a dying linked -//! peer. `Normal`, `Shutdown` and `Kill` are direct requests, -//! which always stop the target (see [`exit`]). Stopping is cooperative: a -//! running handler finishes first, then `stopped()` runs. +//! [`exit`] sends an exit signal with a chosen [`ExitReason`], exactly as +//! Erlang's `exit_signal(Pid, Reason)` (`ChildHandle::exit`). Stopping is +//! cooperative: a running handler finishes first. //! //! Message-level faults (delay, discard, crash on a message, a reply slower //! than the caller's timeout) are configured on [`Mock`](crate::tasks::Mock) //! and [`Probe`](crate::tasks::Probe). -//! -//! Panic injection uses a short-lived linked actor. Kill uses the core's -//! direct `ChildHandle::stop_with_reason`, including in Tokio-free builds. -use spawned_concurrency::{message::Message, threads, ChildHandle, ExitReason}; +use spawned_concurrency::{ChildHandle, ExitReason}; -/// Make `target` exit with `reason`. +/// Send `target` an exit signal with `reason` from outside any actor +/// ([`ChildHandle::exit`], sender `ActorId::EXTERNAL`): /// -/// - `Normal`: [`ChildHandle::stop`]. The target stops even if it traps -/// exits; this differs from Erlang's `exit(Pid, normal)`, which a -/// non-trapping process ignores and a trapping one receives as a message. -/// - `Shutdown`: [`ChildHandle::shutdown`]. Also a direct stop that trapping -/// cannot intercept. -/// - `Panic(msg)`: a short-lived linked actor panics with `msg`. A -/// non-trapping target exits with `Panic("panic in handler: {msg}")` (the -/// wording the core uses for a handler panic), not `Panic(msg)`; a target -/// that traps exits receives an [`Exit`](spawned_concurrency::Exit) carrying -/// that reason instead of stopping. The panic is real, so the default panic -/// hook prints `msg` to stderr. If the target already exited abnormally, -/// its link stops the fuse first; that is logged and `exit` returns (the -/// target is gone, with its own reason). Any other failure to deliver the -/// panic panics here rather than dropping the fault silently. -/// - `Kill`: direct `ChildHandle::stop_with_reason(Kill)`. The target stops -/// even when trapping exits; linked trapping peers receive an Exit notice. -/// Available in both runtime modes without constructing a private runtime. +/// - `Kill`: untrappable; the target ends `Killed` ([`ChildHandle::kill`]). +/// - `Normal`: ignored by a non-trapping target; a trapping one receives an +/// [`Exit`](spawned_concurrency::Exit). +/// - any other reason: a non-trapping target exits with exactly that reason +/// after its current handler, without `stopped()`; a trapping one receives +/// an `Exit` carrying it. /// -/// A target that has already exited is left alone (logged at debug level). A -/// target that exits `Normal` or `Shutdown` concurrently with a `Panic` -/// injection does not propagate that exit over the link, so the fuse still -/// panics on its own and the default hook prints it; the target is unaffected. +/// Use [`ChildHandle::stop`] or [`ChildHandle::stop_with_reason`] for an +/// administrative stop that `trap_exit` cannot intercept and that runs +/// `stopped()`. /// -/// Never waits for the target and never blocks: it returns once the exit is -/// requested, so it is safe to call from async code, including a -/// `current_thread` runtime. Wait for the exit with `ChildHandle::wait_exit_async`, -/// [`tasks::exit_within`](crate::tasks::exit_within) or -/// [`threads::exit_within`](crate::threads::exit_within). +/// A target past its death point is left alone. Never waits and never +/// blocks, so it is safe to call from async code, including a +/// `current_thread` runtime. Wait for the exit with +/// `ChildHandle::wait_exit_async`, [`tasks::exit_within`](crate::tasks::exit_within) +/// or [`threads::exit_within`](crate::threads::exit_within). pub fn exit(target: &ChildHandle, reason: ExitReason) { tracing::debug!(target = ?target.id(), %reason, "spawned-test injecting exit"); - match reason { - ExitReason::Normal => target.stop(), - ExitReason::Shutdown => target.shutdown(), - ExitReason::Panic(msg) => panic_linked(target, msg), - ExitReason::Kill => target.stop_with_reason(ExitReason::Kill), - } -} - -struct Fuse; -impl threads::Actor for Fuse {} -struct Detonate(String); -impl Message for Detonate { - type Result = (); -} -impl threads::Handler for Fuse { - fn handle(&mut self, msg: Detonate, _: &threads::Context) { - panic!("{}", msg.0); - } -} - -fn panic_linked(target: &ChildHandle, msg: String) { - use threads::ActorStart as _; - if already_exited(target) { - return; - } - let fuse = Fuse.start(); - // Registers both link edges synchronously, or, if the target has already - // exited abnormally, signals that exit to the fuse right away. - fuse.context().link(target); - if let Err(error) = fuse.send(Detonate(msg)) { - // Nothing else knows the fuse, so only the target's link can have - // stopped it: the target exited abnormally before the fault landed. - // Read the outcome without waiting (the fuse's mailbox can close - // before its exit reason is published), so async callers never block. - match fuse.child_handle().exit_reason() { - Some(ExitReason::Normal) => panic!( - "spawned-test: could not inject a panic into {:?}: the fuse stopped \ - ({error}) without a linked exit", - target.id() - ), - reason => tracing::debug!( - target = ?target.id(), - fuse_reason = ?reason, - "spawned-test: target exited before the injected panic; not needed" - ), - } - } -} - -/// `true` (and logged) if `target` already exited, so there is nothing to fault. -fn already_exited(target: &ChildHandle) -> bool { - match target.exit_reason() { - Some(reason) => { - tracing::debug!( - target = ?target.id(), - %reason, - "spawned-test: target already exited; fault not injected" - ); - true - } - None => false, - } + target.exit(reason); } diff --git a/testing/src/inbox.rs b/testing/src/inbox.rs index dcc338298..b8054a307 100644 --- a/testing/src/inbox.rs +++ b/testing/src/inbox.rs @@ -1,11 +1,14 @@ //! Shared message log behind mocks and probes: blocking and async waiters. +use spawned_concurrency::message::Message; use std::any::{Any, TypeId}; +use std::collections::HashMap; use std::panic::Location; use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::time::{Duration, Instant}; /// What a mock or probe did with one message it received. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum Outcome { /// Received, but its handling has not finished: the reply closure or a /// `delay` is still running. `expect` and `take_all` skip it until it is @@ -114,6 +117,8 @@ impl Drop for Pending { #[derive(Default)] pub(crate) struct Inbox { + /// Deferred reply tokens by message type (`Mock::defer`). + deferred: Mutex>>>, entries: Mutex>, arrived: Condvar, #[cfg(feature = "async")] @@ -130,6 +135,29 @@ impl Default for AsyncSignal { } impl Inbox { + pub(crate) fn store_deferred( + &self, + token: spawned_concurrency::ReplyTo, + ) { + self.deferred + .lock() + .unwrap_or_else(|p| p.into_inner()) + .entry(std::any::TypeId::of::()) + .or_default() + .push(Box::new(token)); + } + pub(crate) fn take_deferred(&self) -> Vec> { + let tokens = self + .deferred + .lock() + .unwrap_or_else(|p| p.into_inner()) + .remove(&std::any::TypeId::of::()) + .unwrap_or_default(); + tokens + .into_iter() + .filter_map(|t| t.downcast().ok().map(|t| *t)) + .collect() + } fn lock(&self) -> MutexGuard<'_, Vec> { self.entries.lock().unwrap_or_else(|p| p.into_inner()) } diff --git a/testing/src/lib.rs b/testing/src/lib.rs index 6e4cb78e9..1749120d0 100644 --- a/testing/src/lib.rs +++ b/testing/src/lib.rs @@ -17,10 +17,10 @@ //! - **Harness helpers**: [`tasks::settle`], [`tasks::exit_within`], //! [`tasks::wait_for_generation`], [`tasks::shutdown_system`] and //! [`tasks::TestSystem`] turn hangs into test failures with a report. -//! - **Fault injection**: [`faults::exit`] stops any actor with a chosen -//! [`ExitReason`](spawned_concurrency::ExitReason) (`Panic` travels -//! through a link, while `Kill` is direct; an injected panic surfaces as -//! `Panic("panic in handler: ..")` and the default panic hook prints it); +//! - **Fault injection**: [`faults::exit`] sends an exact exit signal with a +//! chosen [`ExitReason`](spawned_concurrency::ExitReason) +//! (`ChildHandle::exit`; `Kill` ends the target `Killed`, `Normal` is +//! ignored by a non-trapping target, a trapping target gets an `Exit`); //! [`tasks::restart_child`] crashes a supervised child and returns its //! replacement; mocks delay, discard or crash on chosen messages. //! @@ -53,3 +53,32 @@ pub mod tasks; pub mod threads; pub use inbox::{Outcome, Received}; + +/// A `PendingReply` and its receiver for calling a manual +/// `Handler::handle` directly in a test, without an actor: +/// +/// ```ignore +/// let (reply, rx) = spawned_test::reply_pair::(); +/// actor.handle(GetCount, reply, &ctx).await; +/// assert_eq!(rx.await.unwrap(), 3); +/// ``` +/// +/// The reply has no deadline. Await (or `recv`) the receiver only after +/// driving every turn the reply depends on: a deferred reply answered by a +/// later message is never answered while the test waits for it first. +pub fn reply_pair() -> ( + spawned_concurrency::PendingReply<'static, R>, + spawned_concurrency::ReplyReceiver, +) { + spawned_concurrency::__private::reply_pair(spawned_concurrency::Deadline::Never) +} + +/// The `PendingReply` a handler receives for a sent message (no caller), +/// for calling a manual `Handler::handle` directly as a send: +/// +/// ```ignore +/// actor.handle(Tick, spawned_test::detached_reply(), &ctx).await; +/// ``` +pub fn detached_reply() -> spawned_concurrency::PendingReply<'static, R> { + spawned_concurrency::__private::detached_reply() +} diff --git a/testing/src/probe.rs b/testing/src/probe.rs index 3a05ffe17..abffbb441 100644 --- a/testing/src/probe.rs +++ b/testing/src/probe.rs @@ -34,6 +34,9 @@ impl Drop for Replying<'_> { } } +// `Receiver` is sealed; the probe is the one implementor outside spawned. +impl spawned_concurrency::__private::Sealed for ProbeCore {} + impl ProbeCore { pub(crate) fn new(reply: Option>) -> Self { Self { @@ -48,7 +51,7 @@ impl ProbeCore { fn open(&self) -> Result, ActorError> { let closed = lock(&self.closed); if *closed { - Err(ActorError::ActorStopped) + Err(ActorError::NoProc) } else { Ok(closed) } @@ -65,7 +68,7 @@ impl ProbeCore { } /// `Ok(None)` means the request was logged but there is no reply: the - /// caller drops the reply sender and observes `ActorStopped`. + /// caller observes `ActorStopped(NoProc)` (no actor stands behind it). /// /// A request is logged `InFlight` on admission. The returned [`Pending`] /// is finished by the caller *after* it has put the reply in the reply diff --git a/testing/src/rules.rs b/testing/src/rules.rs index d7d46a84e..4bd1d934d 100644 --- a/testing/src/rules.rs +++ b/testing/src/rules.rs @@ -42,12 +42,19 @@ pub(crate) enum Plan { record: Option, }, Crash(String), + /// `Mock::defer`: take the reply obligation and store it for the test. + Defer, + /// `Mock::discard`: wait `delay`, then drop the reply obligation. + Discard { + delay: Option, + }, } #[derive(Default)] pub(crate) struct Rules { by_type: HashMap>, allow_sends: bool, + deferred: std::collections::HashSet, } /// `Some(())` as `M::Result` when that type is `()`: lets one generic handler @@ -81,6 +88,9 @@ impl Rules { pub(crate) fn crash_on(&mut self, reason: String) { self.rule::().crash = Some(reason); } + pub(crate) fn defer(&mut self) { + self.deferred.insert(TypeId::of::()); + } pub(crate) fn allow_sends(&mut self) { self.allow_sends = true; } @@ -101,6 +111,10 @@ impl Rules { record: None, }; } + if self.deferred.contains(&TypeId::of::()) { + inbox.push(msg, Outcome::Handled); + return Plan::Defer; + } let name = std::any::type_name::(); let allow_sends = self.allow_sends; let rule = self @@ -139,14 +153,10 @@ impl Rules { return Plan::Crash(reason); } if rule.discard { - let value = unit_result::().expect("discard requires Result = ()"); - let mut record = inbox.arrive(msg); - record.replied(Outcome::Discarded); - return Plan::Reply { - delay, - value, - record: Some(record), - }; + // Discarded: the reply obligation is dropped unanswered, so a + // request gets `ReplyDropped`, as a lost message would. + inbox.push(msg, Outcome::Discarded); + return Plan::Discard { delay }; } if rule.reply.is_none() && unit_result::().is_none() { inbox.push(msg, Outcome::Unexpected); diff --git a/testing/src/shared.rs b/testing/src/shared.rs index 7942d3181..ba7124412 100644 --- a/testing/src/shared.rs +++ b/testing/src/shared.rs @@ -1,13 +1,21 @@ //! Items identical in the `tasks` and `threads` modules, expanded in each. //! -//! The two modes differ in their actor traits, `oneshot` channels and whether +//! The two modes differ in their actor traits and whether //! waits are `async`; everything else about mocks and probes is written once //! here. Each macro resolves `Mock`, `MockRef`, `Probe`, `Recipient`, -//! `Receiver`, `oneshot` and friends in the module that invokes it. +//! `Receiver` and friends in the module that invokes it. /// Builder methods of `Mock`, except `delay` and `start`. macro_rules! mock_builder { () => { + /// Defer every `M` request: the mock takes its reply obligation + /// (`PendingReply::defer`) and stores the token; take it with + /// `MockRef::deferred` to reply later, drop it, or never reply. + pub fn defer(mut self) -> Self { + self.rules.defer::(); + self + } + /// A strict mock: every message type needs a rule. /// /// The receive log keeps one entry per message for the mock's @@ -54,11 +62,11 @@ macro_rules! mock_builder { /// Fault: log each `M` as [`crate::Outcome::Discarded`] and do nothing /// else, as if the message were lost after a successful send. /// - /// Meant for fire-and-forget messages. A handler cannot tell a send - /// from a request, so a *request* `M` (a protocol method returning - /// `Response<()>`) still gets its `()` reply and the caller sees - /// success, whereas a genuinely lost request would time out; use - /// [`delay`](Self::delay) beyond the caller's timeout for that. A + /// Meant for fire-and-forget messages. A *request* `M` (a protocol + /// method returning `Response<()>`) has its reply obligation dropped + /// unanswered, so the caller gets `ReplyDropped`; use + /// [`delay`](Self::delay) beyond the caller's timeout to simulate a + /// request that is never answered. A /// discarded message stays `Discarded` even if the mock dies during /// a `delay` on it. pub fn discard>(mut self) -> Self { @@ -67,7 +75,8 @@ macro_rules! mock_builder { } /// Fault: panic with `reason` on each `M`. The mock exits with - /// `ExitReason::Panic` and pending callers see `ActorStopped`. + /// `ExitReason::Panic` and pending callers see `ActorStopped` with that + /// `Panic` reason. pub fn crash_on(mut self, reason: impl Into) -> Self { self.rules.crash_on::(reason.into()); self @@ -117,6 +126,13 @@ macro_rules! mock_ref_common { self.inbox.count::() } + /// Take the reply tokens of the `M` requests the mock deferred + /// (`Mock::defer`), oldest first: reply later, drop them + /// (`ReplyDropped`), or keep them unanswered. + pub fn deferred(&self) -> Vec> { + self.inbox.take_deferred::() + } + /// Take every not-yet-expected `M`, oldest first. pub fn take_all(&self) -> Vec { self.inbox.take_all::() @@ -152,12 +168,21 @@ macro_rules! probe_common { fn send(&self, msg: M) -> Result<(), ActorError> { ProbeCore::send(self, msg) } - fn request_raw(&self, msg: M) -> Result, ActorError> { - let (tx, rx) = oneshot::channel(); - if let Some((value, record)) = self.request(msg)? { - let _ = tx.send(value); - // Logged `Handled` only once the reply is in the channel. - record.finish(); + fn send_request( + &self, + msg: M, + deadline: Deadline, + ) -> Result, ActorError> { + let (reply, rx) = spawned_concurrency::__private::reply_pair(deadline); + match self.request(msg)? { + Some((value, record)) => { + reply.reply(value); + // Logged `Handled` only once the reply is in the slot. + record.finish(); + } + // An unexpected request is discharged as `ActorStopped` + // (`NoProc`: no actor stands behind a probe). + None => spawned_concurrency::__private::discharge_stopped(reply), } Ok(rx) } @@ -209,8 +234,8 @@ macro_rules! probe_common { self.core.inbox.take_all::() } - /// Fault: behave like a stopped actor; later sends and requests - /// return `ActorStopped`. + /// Fault: behave like an actor past its death point; later sends + /// and requests return `NoProc`. pub fn close(&self) { self.core.close(); } diff --git a/testing/src/tasks.rs b/testing/src/tasks.rs index 21b691544..854bdceef 100644 --- a/testing/src/tasks.rs +++ b/testing/src/tasks.rs @@ -21,7 +21,8 @@ use spawned_concurrency::tasks::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_rt::tasks::{oneshot, sleep, timeout}; +use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; +use spawned_rt::tasks::{sleep, timeout}; use std::future::Future; use std::ops::Deref; use std::panic::Location; @@ -93,8 +94,14 @@ impl Mock { impl Actor for Mock {} impl Handler for Mock { - fn handle(&mut self, msg: M, _: &Context) -> impl Future + Send { + fn handle( + &mut self, + msg: M, + reply: PendingReply<'_, M::Result>, + _: &Context, + ) -> impl Future + Send { let plan = self.rules.plan(&self.inbox, msg); + let inbox = self.inbox.clone(); async move { match plan { Plan::Reply { @@ -108,7 +115,18 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - value + reply.reply(value); + } + Plan::Discard { delay } => { + if let Some(delay) = delay { + sleep(delay).await; + } + drop(reply); + } + Plan::Defer => { + if let Ok(token) = reply.defer() { + inbox.store_deferred::(token); + } } Plan::Crash(reason) => panic!("{reason}"), } @@ -234,22 +252,55 @@ where } } -/// Wait for `handle` to exit, or panic after `deadline` instead of hanging. +/// Run `f` inside one of `actor`'s turns, with its state and `Context`, and +/// return its result: a test reaches `ctx` (to start a job, defer a reply, +/// arm a timer) or inspects state without an ad hoc message or a +/// `cfg(test)` bypass. Messages sent to `actor` earlier are handled first. +/// Panics if the actor is gone or `f` panicked (which crashes the actor, as +/// any handler panic does). +/// +/// ```ignore +/// let handle = in_actor(&server, |server, ctx| server.start_checks(ctx, id)).await; +/// ``` +#[track_caller] +pub fn in_actor(actor: &ActorRef, f: F) -> impl Future + '_ +where + A: Actor, + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, +{ + let at = Location::caller(); + async move { + let reply = match spawned_concurrency::__private::in_turn(actor, f) { + Ok(rx) => rx.await, + Err(error) => Err(error), + }; + reply.unwrap_or_else(|error| { + panic!( + "in_actor on {:?} failed: {error}; exit reason {:?} (called at {at})", + actor.id(), + actor.exit_reason() + ) + }) + } +} + +/// Wait for `handle` to exit, or panic after `limit` instead of hanging. /// Works for actors of either mode. #[track_caller] -pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> impl Future { - exit_within_at(handle.clone(), deadline, Location::caller()) +pub fn exit_within(handle: &ChildHandle, limit: Duration) -> impl Future { + exit_within_at(handle.clone(), limit, Location::caller()) } async fn exit_within_at( handle: ChildHandle, - deadline: Duration, + limit: Duration, at: &'static Location<'static>, ) -> ExitReason { - match timeout(deadline, handle.wait_exit_async()).await { + match timeout(limit, handle.wait_exit_async()).await { Ok(reason) => reason, Err(_) => panic!( - "actor {:?} did not exit within {deadline:?}; a handler may be blocked or the \ + "actor {:?} did not exit within {limit:?}; a handler may be blocked or the \ actor was never asked to stop (exit_within called at {at})", handle.id() ), @@ -260,9 +311,9 @@ async fn exit_within_at( #[track_caller] pub fn join_within( actor: &ActorRef, - deadline: Duration, + limit: Duration, ) -> impl Future { - exit_within_at(actor.child_handle(), deadline, Location::caller()) + exit_within_at(actor.child_handle(), limit, Location::caller()) } /// Poll `condition` every millisecond until it holds; panic naming `what` after `timeout`. @@ -287,16 +338,16 @@ pub fn wait_until<'a>( } /// Wait for a ready generation strictly greater than `after`, or panic with -/// the child's snapshot after `deadline`. +/// the child's snapshot after `limit`. #[track_caller] pub fn wait_for_generation( child: &SupervisedChild, after: u64, - deadline: Duration, + limit: Duration, ) -> impl Future> + '_ { let at = Location::caller(); async move { - match timeout(deadline, child.wait_for_generation(after)).await { + match timeout(limit, child.wait_for_generation(after)).await { Ok(Ok(actor)) => actor, Ok(Err(unavailable)) => panic!( "supervised child will not reach a generation after {after}: {unavailable}; \ @@ -305,7 +356,7 @@ pub fn wait_for_generation( ), Err(_) => panic!( "supervised child did not reach a generation after {after} within \ - {deadline:?}; snapshot {:?} (wait_for_generation called at {at})", + {limit:?}; snapshot {:?} (wait_for_generation called at {at})", child.snapshot() ), } @@ -314,22 +365,22 @@ pub fn wait_for_generation( /// Fault: make the current instance of `child` exit with `reason` (see /// [`faults::exit`]) and return its replacement once the supervisor has started -/// it, or panic after `deadline` (for example when the restart mode does not -/// restart that reason). `deadline` bounds the whole call: finding a running +/// it, or panic after `limit` (for example when the restart mode does not +/// restart that reason). `limit` bounds the whole call: finding a running /// instance, injecting the fault and waiting for the replacement. /// /// The instance targeted and the generation waited past always belong /// together, even if the child restarts on its own meanwhile, so the returned /// reference is the first replacement of the targeted instance. A later /// instance is never faulted: the fault is aimed at the targeted instance's -/// own handle, and a `Kill` is committed before `faults::exit` returns. If the +/// own handle, and the signal is applied before `faults::exit` returns. If the /// targeted instance dies of its own cause first, the fault is a no-op and the /// replacement after that death is returned. #[track_caller] pub fn restart_child( child: &SupervisedChild, reason: ExitReason, - deadline: Duration, + limit: Duration, ) -> impl Future> + '_ { let at = Location::caller(); async move { @@ -341,7 +392,7 @@ pub fn restart_child( faulted.set(Some(generation)); child.wait_for_generation(generation).await }; - let outcome = timeout(deadline, run).await; + let outcome = timeout(limit, run).await; match (outcome, faulted.get()) { (Ok(Ok(actor)), _) => actor, (Ok(Err(unavailable)), generation) => panic!( @@ -350,13 +401,13 @@ pub fn restart_child( child.snapshot() ), (Err(_), None) => panic!( - "supervised child never had a stable instance to fault within {deadline:?}; \ + "supervised child never had a stable instance to fault within {limit:?}; \ snapshot {:?} (restart_child called at {at})", child.snapshot() ), (Err(_), Some(generation)) => panic!( "supervised child did not reach a generation after {generation} within \ - {deadline:?} (one deadline for the whole call); snapshot {:?} \ + {limit:?} (one limit for the whole call); snapshot {:?} \ (restart_child called at {at})", child.snapshot() ), @@ -394,24 +445,24 @@ async fn faultable_instance(child: &SupervisedChild) -> (ActorRef SupervisorReport { - let report = supervisor.shutdown(deadline).await; +/// Shut a supervisor down within `grace`, or panic with its report. +pub async fn shutdown_supervisor(supervisor: &Supervisor, grace: Duration) -> SupervisorReport { + let report = supervisor.shutdown(grace).await; assert!( report.complete, - "supervisor did not shut down within {deadline:?}: failure {:?}, children {:#?}", + "supervisor did not shut down within {grace:?}: failure {:?}, children {:#?}", report.failure, report.children ); report } -/// Shut an [`ActorSystem`] down within `deadline`, or panic with a report of +/// Shut an [`ActorSystem`] down within `grace`, or panic with a report of /// what is still running instead of hanging the test. -pub async fn shutdown_system(system: &ActorSystem, deadline: Duration) -> ShutdownReport { - let report = system.shutdown(deadline).await; +pub async fn shutdown_system(system: &ActorSystem, grace: Duration) -> ShutdownReport { + let report = system.shutdown(grace).await; assert!( report.complete, - "ActorSystem did not shut down within {deadline:?}\n unfinished groups: {:?}\n \ + "ActorSystem did not shut down within {grace:?}\n unfinished groups: {:?}\n \ unfinished actors: {:?}\n exits: {:?}\n supervisor failures: {:?}\n \ group failures: {:?}", report.unfinished_groups, @@ -463,9 +514,9 @@ impl TestSystem { .expect("start mock on test group") } - /// [`shutdown_system`]: panics with a report if not complete by `deadline`. - pub async fn shutdown(self, deadline: Duration) -> ShutdownReport { - shutdown_system(&self.system, deadline).await + /// [`shutdown_system`]: panics with a report if not complete by `grace`. + pub async fn shutdown(self, grace: Duration) -> ShutdownReport { + shutdown_system(&self.system, grace).await } } diff --git a/testing/src/threads.rs b/testing/src/threads.rs index c452bdf50..9bd312451 100644 --- a/testing/src/threads.rs +++ b/testing/src/threads.rs @@ -10,7 +10,7 @@ use spawned_concurrency::threads::{ Actor, ActorRef, ActorStart, Context, Handler, Receiver, Recipient, }; use spawned_concurrency::{ChildHandle, ExitReason}; -use spawned_rt::threads::oneshot; +use spawned_concurrency::{Deadline, PendingReply, ReplyReceiver}; use std::ops::Deref; use std::panic::Location; use std::sync::Arc; @@ -52,7 +52,7 @@ impl Mock { impl Actor for Mock {} impl Handler for Mock { - fn handle(&mut self, msg: M, _: &Context) -> M::Result { + fn handle(&mut self, msg: M, reply: PendingReply<'_, M::Result>, _: &Context) { match self.rules.plan(&self.inbox, msg) { Plan::Reply { delay, @@ -65,7 +65,18 @@ impl Handler for Mock { if let Some(record) = record { record.finish(); } - value + reply.reply(value); + } + Plan::Discard { delay } => { + if let Some(delay) = delay { + std::thread::sleep(delay); + } + drop(reply); + } + Plan::Defer => { + if let Ok(token) = reply.defer() { + self.inbox.store_deferred::(token); + } } Plan::Crash(reason) => panic!("{reason}"), } @@ -158,20 +169,43 @@ where } } -/// Block until `handle` exits, or panic after `deadline` instead of hanging. +/// Run `f` inside one of `actor`'s turns, with its state and `Context`, and +/// return its result: a test reaches `ctx` (to start a job, defer a reply, +/// arm a timer) or inspects state without an ad hoc message or a +/// `cfg(test)` bypass. Messages sent to `actor` earlier are handled first. +/// Panics if the actor is gone or `f` panicked (which crashes the actor, as +/// any handler panic does). +#[track_caller] +pub fn in_actor(actor: &ActorRef, f: F) -> R +where + A: Actor, + R: Send + 'static, + F: FnOnce(&mut A, &Context) -> R + Send + 'static, +{ + let reply = spawned_concurrency::__private::in_turn_threads(actor, f).and_then(|rx| rx.recv()); + reply.unwrap_or_else(|error| { + panic!( + "in_actor on {:?} failed: {error}; exit reason {:?}", + actor.id(), + actor.exit_reason() + ) + }) +} + +/// Block until `handle` exits, or panic after `limit` instead of hanging. /// /// Polls every millisecond: `ChildHandle` has no blocking wait with a /// timeout, and a waiter thread per call would leak on timeout. #[track_caller] -pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> ExitReason { - let end = Instant::now() + deadline; +pub fn exit_within(handle: &ChildHandle, limit: Duration) -> ExitReason { + let end = Instant::now() + limit; loop { if let Some(reason) = handle.exit_reason() { return reason; } if Instant::now() >= end { panic!( - "actor {:?} did not exit within {deadline:?}; a handler may be blocked or the \ + "actor {:?} did not exit within {limit:?}; a handler may be blocked or the \ actor was never asked to stop", handle.id() ); @@ -182,8 +216,8 @@ pub fn exit_within(handle: &ChildHandle, deadline: Duration) -> ExitReason { /// [`exit_within`] for a native `ActorRef`. #[track_caller] -pub fn join_within(actor: &ActorRef, deadline: Duration) -> ExitReason { - exit_within(&actor.child_handle(), deadline) +pub fn join_within(actor: &ActorRef, limit: Duration) -> ExitReason { + exit_within(&actor.child_handle(), limit) } /// Poll `condition` every millisecond until it holds; panic naming `what` after `timeout`. diff --git a/testing/tests/tasks_kit.rs b/testing/tests/tasks_kit.rs index 97102509b..0ece8f97f 100644 --- a/testing/tests/tasks_kit.rs +++ b/testing/tests/tasks_kit.rs @@ -112,7 +112,10 @@ async fn protocol_methods_run_directly_on_the_mock() { async fn unconfigured_request_fails_loudly() { let mock = Mock::new().allow_sends().start(); let error = mock.reserve("x".into()).await.unwrap_err(); - assert!(matches!(error, ActorError::ActorStopped)); + assert!(matches!( + error, + ActorError::ActorStopped(ExitReason::Panic(_)) + )); let ExitReason::Panic(reason) = join_within(&mock, WAIT).await else { panic!("mock should panic on an unconfigured request"); }; @@ -207,7 +210,11 @@ async fn replying_probe_answers_and_closed_probe_rejects() { let error = tasks::request(&*silent.recipient(), Quote(1), WAIT) .await .unwrap_err(); - assert!(matches!(error, ActorError::ActorStopped)); + // No actor stands behind a probe: its reason is NoProc. + assert!(matches!( + error, + ActorError::ActorStopped(ExitReason::NoProc) + )); assert_eq!(silent.received()[0].outcome, Outcome::Unexpected); probe.close(); @@ -225,13 +232,24 @@ impl Message for Get { } impl Actor for Counter {} impl Handler for Counter { - async fn handle(&mut self, _: Bump, _: &Context) { + async fn handle( + &mut self, + _: Bump, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.fetch_add(1, Ordering::SeqCst); + reply.reply(()); } } impl Handler for Counter { - async fn handle(&mut self, _: Get, _: &Context) -> usize { - self.0.load(Ordering::SeqCst) + async fn handle( + &mut self, + _: Get, + reply: spawned_concurrency::PendingReply<'_, usize>, + _: &Context, + ) { + reply.reply(self.0.load(Ordering::SeqCst)); } } @@ -251,25 +269,27 @@ async fn settle_waits_for_earlier_sends() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn exit_injects_every_reason() { - for reason in [ - ExitReason::Normal, - ExitReason::Shutdown, - ExitReason::Panic("injected".into()), - ExitReason::Kill, + // Exact exit_signal/2 semantics: the target ends with the signal's reason, + // a kill ends it Killed. + for (reason, expected) in [ + (ExitReason::Shutdown, ExitReason::Shutdown), + ( + ExitReason::Panic("injected".into()), + ExitReason::Panic("injected".into()), + ), + (ExitReason::Kill, ExitReason::Killed), ] { let actor = Counter(Arc::default()).start(); - faults::exit(&actor.child_handle(), reason.clone()); - let observed = join_within(&actor, WAIT).await; - match reason { - ExitReason::Panic(msg) => { - assert_eq!( - observed, - ExitReason::Panic(format!("panic in handler: {msg}")) - ) - } - other => assert_eq!(observed, other), - } + faults::exit(&actor.child_handle(), reason); + assert_eq!(join_within(&actor, WAIT).await, expected); } + // Normal is ignored by a non-trapping target. + let actor = Counter(Arc::default()).start(); + faults::exit(&actor.child_handle(), ExitReason::Normal); + assert_eq!(settle(&actor, Get, WAIT).await, 0); + assert_eq!(actor.child_handle().exit_reason(), None); + actor.context().stop(); + assert_eq!(join_within(&actor, WAIT).await, ExitReason::Normal); } #[tokio::test] @@ -284,15 +304,14 @@ async fn supervised_child_restarts_after_injected_crash() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::ZERO, - maximum: Duration::ZERO, - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::ZERO) + .maximum(Duration::ZERO) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { @@ -330,10 +349,16 @@ impl Message for Hang { } impl Actor for Stuck {} impl Handler for Stuck { - async fn handle(&mut self, _: Hang, _: &Context) { + async fn handle( + &mut self, + _: Hang, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { self.0.store(true, Ordering::SeqCst); // Blocking code is never preempted, so shutdown cannot finish in time. std::thread::sleep(Duration::from_millis(500)); + reply.reply(()); } } @@ -393,9 +418,10 @@ async fn settle_within_uses_the_given_bound() { .delay::(Duration::from_millis(500)) .start(); // Occupy the mock; the fence queues behind the slow reply. - let _pending = mock - .actor_ref() - .request_raw(inventory_protocol::Reserve { item: "x".into() }); + let _pending = mock.actor_ref().send_request( + inventory_protocol::Reserve { item: "x".into() }, + spawned_concurrency::DEFAULT_REQUEST_TIMEOUT, + ); mock.settle_within(Duration::from_millis(20)).await; } @@ -481,15 +507,14 @@ async fn restart_child_waits_out_a_restart_in_progress() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(100), - maximum: Duration::from_millis(100), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(100)) + .maximum(Duration::from_millis(100)) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { @@ -526,7 +551,7 @@ async fn exit_injection_never_blocks_a_current_thread_runtime() { let observed = join_within(&actor, WAIT).await; assert!(matches!( (&reason, &observed), - (ExitReason::Panic(_), ExitReason::Panic(_)) | (ExitReason::Kill, ExitReason::Kill) + (ExitReason::Panic(_), ExitReason::Panic(_)) | (ExitReason::Kill, ExitReason::Killed) )); // A second injection into the exited target is a non-blocking no-op. faults::exit(&actor.child_handle(), reason); @@ -537,20 +562,19 @@ async fn exit_injection_never_blocks_a_current_thread_runtime() { // Regression: `deadline` bounds the whole call, not each of its phases. Each // phase here waits out a 400 ms backoff, so two phases exceed 600 ms. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[should_panic(expected = "one deadline for the whole call")] +#[should_panic(expected = "one limit for the whole call")] async fn restart_child_has_one_deadline_for_the_whole_call() { let test = TestSystem::new(); let mut tree = test .system() - .supervisor(SupervisorConfig { - backoff: Backoff { - initial: Duration::from_millis(400), - maximum: Duration::from_millis(400), - jitter: false, - ..Backoff::default() - }, - ..SupervisorConfig::default() - }) + .supervisor( + SupervisorConfig::default().backoff( + Backoff::default() + .initial(Duration::from_millis(400)) + .maximum(Duration::from_millis(400)) + .jitter(false), + ), + ) .unwrap(); let child = tree .add(ChildSpec::new("counter", test.group(), |_| async { @@ -577,18 +601,56 @@ async fn async_expect_names_its_caller() { } #[tokio::test] -async fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping_peers() { - for reason in [ExitReason::Kill, ExitReason::Shutdown] { - let target = Counter(Arc::default()).start(); - let observer = Counter(Arc::default()).start(); - target.context().trap_exit(true); - observer.context().trap_exit(true); - observer.context().link(&target.child_handle()); - faults::exit(&target.child_handle(), reason.clone()); - assert_eq!(join_within(&target, WAIT).await, reason); - assert_eq!(settle(&observer, Get, WAIT).await, 0); - assert_eq!(observer.child_handle().exit_reason(), None); - observer.context().stop(); - assert_eq!(join_within(&observer, WAIT).await, ExitReason::Normal); +async fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { + // A kill is untrappable: the trapping target ends Killed and its trapping + // peer receives the (trappable) Killed notice and survives. + let target = Counter(Arc::default()).start(); + let observer = Counter(Arc::default()).start(); + target.context().trap_exit(true); + observer.context().trap_exit(true); + observer.context().link(&target.child_handle()); + faults::exit(&target.child_handle(), ExitReason::Kill); + assert_eq!(join_within(&target, WAIT).await, ExitReason::Killed); + assert_eq!(settle(&observer, Get, WAIT).await, 0); + assert_eq!(observer.child_handle().exit_reason(), None); + observer.context().stop(); + assert_eq!(join_within(&observer, WAIT).await, ExitReason::Normal); + // Any other signal to a trapping target is queued as an Exit. + let target = Counter(Arc::default()).start(); + target.context().trap_exit(true); + faults::exit(&target.child_handle(), ExitReason::Shutdown); + assert_eq!(settle(&target, Get, WAIT).await, 0); + assert_eq!(target.child_handle().exit_reason(), None); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); + assert_eq!(join_within(&target, WAIT).await, ExitReason::Shutdown); +} + +/// `in_actor` runs a closure in one of the actor's turns, after the messages +/// sent before it, with the state and the `Context`. +#[tokio::test] +async fn in_actor_runs_a_closure_in_a_turn() { + struct Tally(u32); + #[actor] + impl Tally { + #[send_handler] + async fn add(&mut self, msg: AddOne, _: &Context) { + self.0 += msg.0; + } } + struct AddOne(u32); + impl Message for AddOne { + type Result = (); + } + let tally = Tally(0).start(); + tally.send(AddOne(2)).unwrap(); + let (seen, own) = spawned_test::tasks::in_actor(&tally, |state, ctx| { + state.0 += 1; + (state.0, ctx.id()) + }) + .await; + assert_eq!((seen, own), (3, tally.id())); + tally.context().stop(); + tally.join().await; } diff --git a/testing/tests/threads_kit.rs b/testing/tests/threads_kit.rs index db50159cc..e0952a20b 100644 --- a/testing/tests/threads_kit.rs +++ b/testing/tests/threads_kit.rs @@ -150,37 +150,38 @@ impl Message for Ping { type Result = (); } impl Handler for Idle { - fn handle(&mut self, _: Ping, _: &Context) {} + fn handle( + &mut self, + _: Ping, + reply: spawned_concurrency::PendingReply<'_, ()>, + _: &Context, + ) { + reply.reply(()); + } } #[test] fn native_exit_injection() { - for reason in [ - ExitReason::Normal, - ExitReason::Shutdown, - ExitReason::Panic("injected".into()), - ] { + for reason in [ExitReason::Shutdown, ExitReason::Panic("injected".into())] { let actor = Idle.start(); settle(&actor, Ping, WAIT); faults::exit(&actor.child_handle(), reason.clone()); - let observed = join_within(&actor, WAIT); - match reason { - ExitReason::Panic(msg) => { - assert_eq!( - observed, - ExitReason::Panic(format!("panic in handler: {msg}")) - ) - } - other => assert_eq!(observed, other), - } + assert_eq!(join_within(&actor, WAIT), reason); } + // Normal is ignored by a non-trapping target. + let actor = Idle.start(); + faults::exit(&actor.child_handle(), ExitReason::Normal); + settle(&actor, Ping, WAIT); + assert_eq!(actor.child_handle().exit_reason(), None); + actor.context().stop(); + assert_eq!(join_within(&actor, WAIT), ExitReason::Normal); } #[test] fn native_kill_injection() { let actor = Idle.start(); faults::exit(&actor.child_handle(), ExitReason::Kill); - assert_eq!(join_within(&actor, WAIT), ExitReason::Kill); + assert_eq!(join_within(&actor, WAIT), ExitReason::Killed); } #[test] @@ -234,7 +235,9 @@ fn native_expect_no_message_reports_in_flight_messages() { q.0 }) .start(); - let _reply = mock.request_raw(Quote(1)).unwrap(); + let _reply = mock + .send_request(Quote(1), spawned_concurrency::DEFAULT_REQUEST_TIMEOUT) + .unwrap(); wait_until(WAIT, "the quote arrived", || mock.count::() == 1); mock.expect_no_message(Duration::from_millis(5)); } @@ -287,9 +290,11 @@ fn native_nothing_is_logged_after_close_returns() { #[test] fn native_panic_injection_into_an_exited_target_is_a_no_op() { let actor = Idle.start(); - actor.child_handle().shutdown(); + actor + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); assert_eq!(join_within(&actor, WAIT), ExitReason::Shutdown); - // The target's link stops the fuse before it detonates: not an error. + // A signal to an actor past its death point is dropped. faults::exit(&actor.child_handle(), ExitReason::Panic("late".into())); } @@ -299,7 +304,7 @@ fn native_kill_injection_is_reliable() { for _ in 0..200 { let actor = Idle.start(); faults::exit(&actor.child_handle(), ExitReason::Kill); - assert_eq!(join_within(&actor, WAIT), ExitReason::Kill); + assert_eq!(join_within(&actor, WAIT), ExitReason::Killed); } } @@ -406,18 +411,52 @@ fn native_helpers_name_their_caller() { } #[test] -fn direct_kill_and_shutdown_stop_trapping_targets_without_killing_trapping_peers() { - for reason in [ExitReason::Kill, ExitReason::Shutdown] { - let target = Idle.start(); - let observer = Idle.start(); - target.context().trap_exit(true); - observer.context().trap_exit(true); - observer.context().link(&target.child_handle()); - faults::exit(&target.child_handle(), reason.clone()); - assert_eq!(join_within(&target, WAIT), reason); - settle(&observer, Ping, WAIT); - assert_eq!(observer.child_handle().exit_reason(), None); - observer.context().stop(); - assert_eq!(join_within(&observer, WAIT), ExitReason::Normal); +fn kill_stops_a_trapping_target_and_other_signals_are_trapped() { + let target = Idle.start(); + let observer = Idle.start(); + target.context().trap_exit(true); + observer.context().trap_exit(true); + observer.context().link(&target.child_handle()); + faults::exit(&target.child_handle(), ExitReason::Kill); + assert_eq!(join_within(&target, WAIT), ExitReason::Killed); + settle(&observer, Ping, WAIT); + assert_eq!(observer.child_handle().exit_reason(), None); + observer.context().stop(); + assert_eq!(join_within(&observer, WAIT), ExitReason::Normal); + let target = Idle.start(); + target.context().trap_exit(true); + faults::exit(&target.child_handle(), ExitReason::Shutdown); + settle(&target, Ping, WAIT); + assert_eq!(target.child_handle().exit_reason(), None); + target + .child_handle() + .stop_with_reason(spawned_concurrency::ExitReason::Shutdown); + assert_eq!(join_within(&target, WAIT), ExitReason::Shutdown); +} + +/// `in_actor` runs a closure in one of the actor's turns, after the messages +/// sent before it, with the state and the `Context`. +#[test] +fn native_in_actor_runs_a_closure_in_a_turn() { + struct Tally(u32); + #[actor] + impl Tally { + #[send_handler] + fn add(&mut self, msg: AddOne, _: &Context) { + self.0 += msg.0; + } } + struct AddOne(u32); + impl Message for AddOne { + type Result = (); + } + let tally = Tally(0).start(); + tally.send(AddOne(2)).unwrap(); + let (seen, own) = spawned_test::threads::in_actor(&tally, |state, ctx| { + state.0 += 1; + (state.0, ctx.id()) + }); + assert_eq!((seen, own), (3, tally.id())); + tally.context().stop(); + tally.join(); } diff --git a/verification/api/release.json b/verification/api/release.json new file mode 100644 index 000000000..e4533acce --- /dev/null +++ b/verification/api/release.json @@ -0,0 +1,5 @@ +{ + "baseline": "v0.7.3", + "release_type": "major", + "reason": "0.8 is a breaking pre-1.0 release (0.7 -> 0.8, the 0.x counterpart of a major release): Handler::handle takes the reply obligation, Receiver gained send_request and is sealed, enums and read-only structs became non_exhaustive, config structs gained builders, and the job, request and deadline APIs were consolidated. Every break is a row of docs/MIGRATION.md's 0.7 to 0.8 call-site table. The tool is told `major`: it then skips its breaking-change lints and only checks that the current and baseline rustdoc APIs build in every profile. Return to release_type patch (baseline v0.8.0) once 0.8.0 is tagged." +} diff --git a/verification/model-map.toml b/verification/model-map.toml index 8e2429d7e..fd19c12dd 100644 --- a/verification/model-map.toml +++ b/verification/model-map.toml @@ -113,9 +113,7 @@ code = [ "concurrency/src/mailbox_admission.rs", "concurrency/src/lifecycle_trace.rs", ] -[models.Lifecycle.tlc_only] -reason = "models the proposed v0.8 exit-signal contract; the code on main is v0.7, so there are no lifecycle trace events yet" -tracking = "PR #240 adds LifeEvent and TraceLifecycle.tla; remove this entry once it merges" +coverage_floor = 17 # of 17 trace-spec arms (trace_coverage.py STATIC) [models.DeferredReplies] # Rank 11: reply slots, deadlines, deferral credits, owner exit and death. @@ -128,9 +126,7 @@ code = [ "concurrency/src/reply/**", "concurrency/src/lifecycle_trace.rs", ] -[models.DeferredReplies.tlc_only] -reason = "rank 11 design model; the implementation is not on main yet" -tracking = "PR #240 adds reply events and TraceReplies.tla; remove this entry once it merges" +coverage_floor = 14 # of 14 action keys (TraceReplies arms) [models.DeferredJobs] # Rank 11: pipe jobs, keyed replacement, ActorId and cancellation fencing, @@ -146,9 +142,7 @@ code = [ "concurrency/src/threads/jobs.rs", "concurrency/src/lifecycle_trace.rs", ] -[models.DeferredJobs.tlc_only] -reason = "rank 11 design model; the implementation is not on main yet" -tracking = "PR #240 adds JobEvent and TraceJobs.tla; remove this entry once it merges" +coverage_floor = 10 # of 10 trace-spec arms (trace_coverage.py STATIC) [models.DynamicSupervisor] # Rank 12a design model (PR #245); no code yet. @@ -245,9 +239,11 @@ emitters = [ specs = ["verification/tla/TraceLifecycle.tla"] [trace.rejected] Held = "a signal held while started() runs (phase 0) is outside Lifecycle.tla; the trace is rejected" +[trace.unlogged] +EarlyFire = "fault device only (Fault = EarlyNotify: links and monitors fire when exit begins); no code path takes it" [trace.translated] Spawn = { into = [], why = "trace_check.py numbers actors from Spawn events" } -Send = { into = ["Request", "Enqueue"], why = "trace_check.py splits Send into Request and Enqueue" } +Send = { into = ["Request", "Enqueue", "Send"], why = "trace_check.py splits a Send from outside code into Request and Enqueue, and keeps one from a recorded actor as the model's Send" } [[trace]] struct = "JobEvent" @@ -255,6 +251,8 @@ emitters = [ { call = '\bJobEvent::new\(', arg = 0 }, ] specs = ["verification/tla/TraceJobs.tla"] +[trace.unlogged] +Restart = "a new owner instance reading the old one's slot-addressed mailbox: the code has no such mailbox (a job's messages go to its owner instance's own mailbox, discarded at that instance's death), so the model's cross-instance delivery is a conservative over-approximation (NoLateResult holds a fortiori); a trace is one instance" [[trace]] struct = "ReplyEvent" diff --git a/verification/tla/Cohorts.tla b/verification/tla/Cohorts.tla index 9d1ea42f3..cf72658da 100644 --- a/verification/tla/Cohorts.tla +++ b/verification/tla/Cohorts.tla @@ -1,6 +1,12 @@ ----------------------------- MODULE Cohorts ----------------------------- -(* Coordinator policy and actual generation ownership. Backoff durations, *) -(* handlers and retry budgets are abstracted; Supervision covers policy. *) +(* Coordinator policy and actual generation ownership. Backoff durations *) +(* and handlers are abstracted; Supervision covers the per-child policy. *) +(* The restart budget is modelled as its charge (rank 5f): a OneForAll or *) +(* RestForOne round takes one grant when the coordinator selects it, its *) +(* rebuilds take none, and a failed rebuild asks for a new round, charged *) +(* again when selected (supervision.rs charge_round). Exhaustion itself is *) +(* Supervision's (BudgetBounded); here RoundCharged pins the charge to the *) +(* rounds. Fault "PerChildCharge" restores v0.7's charge per rebuilt child.*) EXTENDS Naturals, FiniteSets, Sequences, TLC (* `@type:` comments are Apalache annotations (CohortsInd.tla); TLC *) (* ignores them. Integer ranges with a state-dependent bound are written *) @@ -32,7 +38,7 @@ ASSUME Mode \in [Children -> {"Permanent", "Transient", "Temporary"}] \* scanned: Set(Int), affected: Set(Int), stopped: Set(Int), \* clean: Set(Int), resumed: Set(Int), ack: Set(Int), terminal: Set(Int), \* gen: Int -> Int, live: Int -> Set(Int), published: Int -> Int, -\* begun: Set(Int), +\* begun: Set(Int), charged: Int, \* badBarrier: Bool, badClosed: Bool, badTerminal: Bool, badRetry: Bool }; CohortTypeAliases == TRUE VARIABLE @@ -48,7 +54,7 @@ Init == affected |-> {}, stopped |-> {}, clean |-> {}, resumed |-> {}, ack |-> {}, terminal |-> {}, gen |-> [c \in Children |-> 0], live |-> [c \in Children |-> {}], published |-> [c \in Children |-> 0], - begun |-> {}, + begun |-> {}, charged |-> 0, badBarrier |-> FALSE, badClosed |-> FALSE, badTerminal |-> FALSE, badRetry |-> FALSE] Range == {j \in 0..N : s.start <= j} \* = s.start..N, since start \in 0..N @@ -64,6 +70,8 @@ CSelect(c,r,n) == /\ r = s.epoch + 1 /\ r <= MaxRounds /\ n = IF Fault = "Selection" THEN c ELSE ExpectedStart(c) /\ s' = [s EXCEPT !.epoch = r, !.origin = c, !.start = n, + \* One restart grant for the whole round, taken at selection. + !.charged = IF Fault = "PerChildCharge" THEN @ ELSE @ + 1, !.covered = [j \in Children |-> IF j >= n THEN r ELSE s.covered[j]], !.pending = {j \in s.pending : j < n}, !.scanned = {}, !.affected = {}, !.stopped = {}, !.clean = {}, @@ -97,9 +105,13 @@ CResume(c,r) == \* requests a new round (CRequest), which OneForAll applies to the whole \* cohort, as OTP's try_again_restart re-runs the strategy. Fault \* "InPlaceRetry" restores the old in-round retry of that child alone. +\* Round 0 (the first start) begins once per child: a failed first start +\* fails the supervisor's start (S1, otp-exit-signals.md). Fault +\* "FirstStartRetry" restores v0.7's retry of it. CBegin(c,r,n) == /\ c \in Active /\ n = s.gen[c] + 1 /\ n <= MaxGen /\ s.open \/ Fault = "Closed" + /\ r = 0 => n = 1 \/ Fault = "FirstStartRetry" /\ \/ /\ r = 0 /\ ~s.committed /\ c \notin s.initial /\ \A j \in Children : j < c => j \in s.initial \/ /\ r = s.assigned[c] /\ r > 0 /\ c \in s.resumed /\ c \notin s.ack @@ -108,6 +120,7 @@ CBegin(c,r,n) == /\ Mode[c] # "Temporary" \/ s.gen[c] = 0 /\ s' = [s EXCEPT !.gen[c] = n, !.live[c] = @ \cup {n}, !.begun = IF r > 0 THEN @ \cup {c} ELSE @, + !.charged = IF Fault = "PerChildCharge" /\ r > 0 THEN @ + 1 ELSE @, !.badBarrier = @ \/ (r > 0 /\ s.clean # s.affected), !.badClosed = @ \/ ~s.open, !.badRetry = @ \/ (r > 0 /\ c \in s.begun)] @@ -155,6 +168,7 @@ TypeOK == /\ s.gen \in [Children -> 0..MaxGen] /\ s.live \in [Children -> SUBSET (1..MaxGen)] /\ s.published \in [Children -> 0..MaxGen] + /\ s.charged \in 0..(MaxRounds * N) AtMostOneGeneration == \A c \in Children : Cardinality(s.live[c]) <= 1 PublishedIsLive == \A c \in Children : s.published[c] # 0 => s.published[c] \in s.live[c] StrategySelection == s.epoch = 0 \/ s.start = ExpectedStart(s.origin) @@ -163,10 +177,16 @@ OneStartPerRound == ~s.badRetry NoStartAfterClose == ~s.badClosed NoRetiredParticipant == ~s.badTerminal TemporaryStartsOnce == \A c \in Children : Mode[c] = "Temporary" => s.gen[c] <= 1 +\* S1: a child that never completed its first start had one attempt. +FirstStartOnce == \A c \in Children : c \notin s.initial => s.gen[c] <= 1 DoneMeansQuiescent == s.phase = "Done" => \A c \in Children : s.live[c] = {} +\* Rank 5f: one restart grant per selected round (superseded ones included), +\* none per rebuilt child. +RoundCharged == s.charged = s.epoch Safety == TypeOK /\ AtMostOneGeneration /\ PublishedIsLive /\ StrategySelection /\ WholeCohortBarrier /\ NoStartAfterClose /\ NoRetiredParticipant /\ TemporaryStartsOnce /\ DoneMeansQuiescent /\ OneStartPerRound + /\ FirstStartOnce /\ RoundCharged Next == \/ \E c \in Children, r \in Epochs, x \in {"true","false"} : CRequest(c,r,x) \/ \E c \in Children, r \in Epochs, n \in Children : CSelect(c,r,n) diff --git a/verification/tla/CohortsInd.tla b/verification/tla/CohortsInd.tla index 9212617e5..97ffcf007 100644 --- a/verification/tla/CohortsInd.tla +++ b/verification/tla/CohortsInd.tla @@ -20,7 +20,7 @@ TypeInd == ack: SUBSET Children, terminal: SUBSET Children, gen: [Children -> 0..MaxGen], live: [Children -> SUBSET (1..MaxGen)], published: [Children -> 0..MaxGen], - begun: SUBSET Children, + begun: SUBSET Children, charged: 0..(MaxRounds * N), badBarrier: BOOLEAN, badClosed: BOOLEAN, badTerminal: BOOLEAN, badRetry: BOOLEAN] @@ -46,7 +46,17 @@ H_barrier == \* false. H_done == s.phase = "Done" => ~s.open -IndInv == TypeInd /\ H_barrier /\ H_done /\ Safety +\* S1: recovery rounds exist only after the tree committed ready, which +\* needs every child's first start. CRequest needs committed; CSelect needs +\* a pending request; CAssign assigns the selected round. So a round-r > 0 +\* CBegin never runs for a child without its first start (FirstStartOnce). +H_first == + /\ s.committed => s.initial = Children + /\ s.pending # {} => s.committed + /\ s.epoch > 0 => s.committed + /\ \A c \in Children : s.assigned[c] > 0 => s.committed + +IndInv == TypeInd /\ H_barrier /\ H_done /\ H_first /\ Safety \* Probe (apalache.sh requires it violated): IndInv admits a mid-round state \* with live generations, i.e. the step check is not over an empty set. diff --git a/verification/tla/Coordinator.tla b/verification/tla/Coordinator.tla index 9b6590456..34224f03f 100644 --- a/verification/tla/Coordinator.tla +++ b/verification/tla/Coordinator.tla @@ -6,6 +6,12 @@ EXTENDS Naturals, FiniteSets (* `targets.iter().rev()`), so "lower index stops first" is "later *) (* admitted stops first". Targets abstract roots whose startup committed; *) (* an unready supervisor startup rolls itself back and is not modelled. *) +(* *) +(* v0.8 (E3): the ordered loop sends each root its parent's Shutdown exit *) +(* signal; when the grace expires, the remaining roots get the kill *) +(* (`killed`), so a trapping root stuck in stopped() cannot hold the join. *) +(* The panic fallback (emergency_shutdown) still stops with Shutdown. A *) +(* supervisor root applies its own per-child kill (Supervision.tla). *) (* `@type:` comments are Apalache annotations (CoordinatorInd.tla); TLC *) (* ignores them. *) CONSTANTS @@ -30,36 +36,47 @@ VARIABLES \* @type: Bool; reported, \* @type: Bool; - released -vars == <> + released, + \* @type: Bool; + expired, + \* @type: Set(Int); + killed +vars == <> Init == /\ phase = "Running" /\ open = TRUE /\ signalled = {} /\ finished = {} /\ index = 1 /\ crashed = FALSE /\ reported = FALSE /\ released = FALSE + /\ expired = FALSE /\ killed = {} Begin == /\ phase = "Running" /\ phase' = "Ordering" /\ open' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> StopNext == /\ phase = "Ordering" /\ index \in Targets /\ signalled' = signalled \cup {index} /\ phase' = "Waiting" - /\ UNCHANGED <> + /\ killed' = IF Fault = "KillEarly" THEN killed \cup {index} ELSE killed + /\ UNCHANGED <> Advance == /\ phase = "Waiting" /\ index \in finished /\ index' = index + 1 /\ phase' = "Ordering" - /\ UNCHANGED <> + /\ UNCHANGED <> Ordered == /\ phase = "Ordering" /\ index > TargetCount /\ phase' = "Joining" - /\ UNCHANGED <> + /\ UNCHANGED <> GraceExpired == /\ phase \in {"Ordering", "Waiting"} /\ phase' = "Emergency" - /\ UNCHANGED <> + /\ expired' = TRUE + /\ UNCHANGED <> Crash == /\ phase # "Done" /\ ~crashed /\ crashed' = TRUE /\ reported' = (Fault # "ForgetFailure") /\ phase' = "Emergency" /\ open' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> +\* After the grace expired (and no panic took the fallback), every root not +\* yet finished gets the kill; otherwise the parent's Shutdown. Emergency == /\ phase = "Emergency" /\ phase' = "Joining" /\ signalled' = IF Fault = "ForgetSignal" THEN signalled ELSE Targets - /\ UNCHANGED <> + /\ killed' = IF expired /\ ~crashed /\ Fault # "NoKillAtGrace" + THEN killed \cup (signalled' \ finished) ELSE killed + /\ UNCHANGED <> Finish(i) == /\ i \in signalled \ finished /\ finished' = finished \cup {i} - /\ UNCHANGED <> + /\ UNCHANGED <> Join == /\ phase = "Joining" /\ (finished = Targets \/ Fault = "ReleaseEarly") /\ phase' = "Done" /\ released' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> \* Negative control for the #209 bug: once admission closed, a supervisor \* startup guard dropped after readiness rolled the (ready) tree back, \* stopping it outside the coordinator's order. The fix only rolls back @@ -67,7 +84,7 @@ Join == /\ phase = "Joining" /\ (finished = Targets \/ Fault = "ReleaseEarly") \* this action is disabled and the model is unchanged. RollbackReady(i) == /\ Fault = "RollbackReady" /\ ~open /\ i \in Targets \ signalled /\ signalled' = signalled \cup {i} - /\ UNCHANGED <> + /\ UNCHANGED <> Next == Begin \/ StopNext \/ Advance \/ Ordered \/ GraceExpired \/ Crash \/ Emergency \/ (\E i \in Targets: Finish(i)) \/ Join \/ (\E i \in Targets: RollbackReady(i)) Safety == /\ phase \in {"Running", "Ordering", "Waiting", "Emergency", "Joining", "Done"} @@ -75,6 +92,7 @@ Safety == /\ phase \in {"Running", "Ordering", "Waiting", "Emergency", "Joining" /\ (phase # "Running" => ~open) /\ (released => finished = Targets /\ phase = "Done") /\ (crashed => reported) + /\ killed \subseteq signalled \* Ordered shutdown: while the coordinator is in its ordered loop, exactly \* the targets before the cursor (and the cursor itself once StopNext ran) \* have been signalled, and every target before the cursor has finished. @@ -89,6 +107,15 @@ OrderedShutdown == \* every target was signalled, whether by the ordered loop or the \* emergency fallback: no root is left running unsignalled behind a join. JoinAfterAllSignalled == phase \in {"Joining", "Done"} => signalled = Targets +\* v0.8: a root is killed only once the grace expired (before it, roots get +\* the parent's Shutdown and may run stopped()). +KillOnlyAtGrace == killed # {} => expired +\* v0.8: after the grace expired without a panic, the join waits only on +\* killed roots: every root has finished or was killed (so a trapping root +\* stuck in stopped() cannot hold runtime ownership). +GraceKillsRemaining == + (phase \in {"Joining", "Done"} /\ expired /\ ~crashed) => + \A i \in Targets : i \in finished \/ i \in killed FairSpec == Init /\ [][Next]_vars /\ WF_vars(Begin) /\ WF_vars(StopNext) /\ WF_vars(Advance) /\ WF_vars(Ordered) /\ WF_vars(Emergency) /\ WF_vars(Join) diff --git a/verification/tla/CoordinatorInd.tla b/verification/tla/CoordinatorInd.tla index 8071ebbf6..f5f99310d 100644 --- a/verification/tla/CoordinatorInd.tla +++ b/verification/tla/CoordinatorInd.tla @@ -20,6 +20,7 @@ TypeInd == /\ finished \in SUBSET Targets /\ index \in 1..TargetCount + 1 /\ crashed \in BOOLEAN /\ reported \in BOOLEAN /\ released \in BOOLEAN + /\ expired \in BOOLEAN /\ killed \in SUBSET Targets ----------------------------------------------------------------------------- (* Helper lemma (the only one needed: without it Apalache returns a CTI *) @@ -34,14 +35,19 @@ TypeInd == \* to Running. Begin then opens the ordered loop at target 1. H_cursor == phase = "Running" => index = 1 +\* The grace expires only in the ordered loop, which it ends: GraceExpired +\* moves to Emergency, from which the loop is never re-entered. +H_expired == expired => phase \in {"Emergency", "Joining", "Done"} + ----------------------------------------------------------------------------- (* The model's state invariants (Coordinator.tla). Safety contains its own *) (* typing conjuncts; OrderedShutdown and JoinAfterAllSignalled are the *) (* ordered-shutdown properties. *) SafetyInd == Safety /\ OrderedShutdown /\ JoinAfterAllSignalled + /\ KillOnlyAtGrace /\ GraceKillsRemaining -Helpers == H_cursor +Helpers == H_cursor /\ H_expired IndInv == TypeInd /\ Helpers /\ SafetyInd diff --git a/verification/tla/CovSupervision.tla b/verification/tla/CovSupervision.tla index a8cbe4910..78a12e696 100644 --- a/verification/tla/CovSupervision.tla +++ b/verification/tla/CovSupervision.tla @@ -42,6 +42,8 @@ CovNext == \/ StopObserved(c) /\ L("StopObserved", c) \/ CleanupDone(c) /\ L("CleanupDone", c) \/ CleanupStuck(c) /\ L("CleanupStuck", c) + \/ DeferExit(c) /\ L("DeferExit", c) + \/ Undefer(c) /\ L("Undefer", c) \/ Decide(c) /\ L("Decide", c) \/ Finish(c) /\ L("Finish", c) diff --git a/verification/tla/DeferredJobs.tla b/verification/tla/DeferredJobs.tla index c7dbaad02..731a1a0ee 100644 --- a/verification/tla/DeferredJobs.tla +++ b/verification/tla/DeferredJobs.tla @@ -4,7 +4,7 @@ (* docs/design/deferred-replies.md. Part 1 is DeferredReplies.tla. *) (* *) (* An owner slot is restarted up to MaxGen times; each instance has a new *) -(* ActorId. Jobs are unlinked (ctx.pipe) or keyed (ctx.pipe_keyed), with *) +(* ActorId. Jobs are unlinked (ctx.pipe) or keyed (JobOptions::key), with *) (* or without a job deadline. Each job has three separate states: *) (* execution (running/finished; cancellation is cooperative, so cancelled *) (* or timed-out work keeps running and keeps its work permit), *) @@ -34,12 +34,39 @@ (* ReplacePendingOnly(NoStaleKeyed) replacement skips notified jobs *) (* EmitAfterTerminal (TerminalLast) emits continue after the terminal *) (* NoEmitCredit (EmitCreditsBounded) *) +(* EarlyCredit (HeldCounted) a held result returns its credit *) +(* when the work ends *) +(* HeldDeadlineLost (DeadlineArmedWhileHeld) the deadline cannot take *) +(* a held result's notification *) +(* ExitKeepsHeld (ExitCancelsJobs) owner exit cancels running jobs *) +(* only, not held results or scheduled emits *) +(* NoAdmissionFence (ExitCancelsJobs) a job is admitted after the exit *) +(* cancelled the job set (admission and cancel_all are *) +(* not serialized under the jobs lock) *) +(* *) +(* Delayed emits (emit_at) and held results: a running job may schedule an *) +(* emit (it takes an emit credit at once and is enqueued, fenced like any *) +(* emit, when it fires). When the work ends while emits are scheduled, its *) +(* result is held: the terminal message is enqueued after the last *) +(* scheduled emit fires, so it stays last. Each job owns one *) +(* MAX_OUTSTANDING credit (`credit`) from admission until its terminal *) +(* message is enqueued, or, when the notification was taken or suppressed *) +(* otherwise, until execution ends: a held result keeps it (HeldCounted). *) +(* The job deadline can take a held result's notification (it stays armed *) +(* until the terminal message). *) +(* *) +(* Releasing a held result is two steps, as in the code: the last fire *) +(* takes the held result under the job's lock (FireScheduled; `releasing`) *) +(* and the terminal message is enqueued after the lock is released *) +(* (ReleaseHeld), unless a deadline or a cancel took or suppressed the *) +(* notification in between. The job keeps its credit until ReleaseHeld. *) (***************************************************************************) EXTENDS Naturals, FiniteSets, Sequences CONSTANTS J, Keyed, JobBound, MaxGen, MaxEmit, Credits, Fault ASSUME Fault \in {"None", "NoInstanceCheck", "NoCancelFence", "DoubleTerminal", "ReleaseOnTimeout", "KeyNoCancel", "ReplacePendingOnly", - "EmitAfterTerminal", "NoEmitCredit"} + "EmitAfterTerminal", "NoEmitCredit", "EarlyCredit", + "HeldDeadlineLost", "ExitKeepsHeld", "NoAdmissionFence"} Jobs == 1..J ASSUME Keyed \subseteq Jobs @@ -55,10 +82,18 @@ VARIABLES started, \* order in which keyed jobs started (for NoStaleKeyed) mbox, \* FIFO sequence of [j, kind] termSent, \* terminal messages enqueued per job - seen \* history of dispatched messages + seen, \* history of dispatched messages + sched, \* scheduled (emit_at) emits not yet fired, each with a credit + held, \* the work ended with emits scheduled: its result is held + credit, \* the job holds its MAX_OUTSTANDING credit + releasing \* the last fire took the held result; its terminal is next vars == <> + started, mbox, termSent, seen, sched, held, credit, releasing>> + +\* Variables of delayed emits and held results, unchanged by the steps that +\* do not concern them. +heldVars == <> Init == /\ inst = 1 /\ alive = TRUE @@ -68,77 +103,166 @@ Init == /\ emitted = [j \in Jobs |-> 0] /\ inflight = [j \in Jobs |-> 0] /\ started = <<>> /\ mbox = <<>> /\ termSent = [j \in Jobs |-> 0] /\ seen = {} + /\ sched = [j \in Jobs |-> 0] /\ held = [j \in Jobs |-> FALSE] + /\ credit = [j \in Jobs |-> FALSE] + /\ releasing = [j \in Jobs |-> FALSE] Running == {j \in Jobs : exec[j] = "running"} -Permits == IF Fault = "ReleaseOnTimeout" THEN {j \in Running : notif[j] = "pending"} - ELSE Running +\* Jobs holding a MAX_OUTSTANDING credit (admission checks the bound). +Permits == {j \in Jobs : credit[j]} LiveKeyed == {j \in Keyed : owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j] /\ notif[j] = "pending"} \* Earlier keyed jobs a replacement must cancel. ToReplace == IF Fault = "ReplacePendingOnly" THEN LiveKeyed ELSE {j \in Keyed : owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j]} -CancelSet(S) == +\* Cancellation drops scheduled emits (their credits return) and a held +\* result, whose run then releases the job's credit. +CancelCore(S) == /\ cancelled' = [j \in Jobs |-> IF j \in S THEN TRUE ELSE cancelled[j]] /\ notif' = [j \in Jobs |-> IF j \in S /\ notif[j] = "pending" THEN "suppressed" ELSE notif[j]] + /\ sched' = [j \in Jobs |-> IF j \in S THEN 0 ELSE sched[j]] + /\ held' = [j \in Jobs |-> IF j \in S THEN FALSE ELSE held[j]] +\* A releasing job keeps its credit until ReleaseHeld. +CreditAfterCancel(S) == + [j \in Jobs |-> IF j \in S /\ exec[j] = "finished" /\ ~releasing[j] THEN FALSE ELSE credit[j]] +CancelSet(S) == CancelCore(S) /\ credit' = CreditAfterCancel(S) +\* Admission and the exit's cancel_all are serialized (one jobs lock and +\* its closed flag): no job is admitted once the exit cancelled the set. Start(j) == - /\ alive /\ exec[j] = "none" /\ Cardinality(Permits) < JobBound + /\ alive \/ Fault = "NoAdmissionFence" + /\ exec[j] = "none" /\ Cardinality(Permits) < JobBound /\ exec' = [exec EXCEPT ![j] = "running"] /\ owner' = [owner EXCEPT ![j] = inst] /\ \E d \in BOOLEAN : hasDl' = [hasDl EXCEPT ![j] = d] - /\ IF j \in Keyed + /\ IF j \in Keyed /\ Fault # "KeyNoCancel" THEN /\ started' = Append(started, j) - /\ IF Fault = "KeyNoCancel" THEN UNCHANGED <> - ELSE CancelSet(ToReplace) - ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ CancelCore(ToReplace) + /\ credit' = [CreditAfterCancel(ToReplace) EXCEPT ![j] = TRUE] + ELSE /\ started' = IF j \in Keyed THEN Append(started, j) ELSE started + /\ credit' = [credit EXCEPT ![j] = TRUE] + /\ UNCHANGED <> + /\ UNCHANGED <> + +Accepting(j) == notif[j] = "pending" /\ ~cancelled[j] CanEmit(j) == /\ exec[j] = "running" /\ emitted[j] < MaxEmit - /\ (notif[j] = "pending" /\ ~cancelled[j]) \/ Fault = "EmitAfterTerminal" - /\ inflight[j] < Credits \/ Fault = "NoEmitCredit" + /\ Accepting(j) \/ Fault = "EmitAfterTerminal" + /\ inflight[j] + sched[j] < Credits \/ Fault = "NoEmitCredit" Emit(j) == /\ CanEmit(j) /\ emitted' = [emitted EXCEPT ![j] = @ + 1] /\ inflight' = [inflight EXCEPT ![j] = @ + 1] /\ mbox' = Append(mbox, [j |-> j, kind |-> "emit"]) - /\ UNCHANGED <> + /\ UNCHANGED <> + +\* emit_at: take an emit credit now; the emit is enqueued when it fires. +ScheduleEmit(j) == + /\ exec[j] = "running" /\ emitted[j] < MaxEmit /\ Accepting(j) + /\ inflight[j] + sched[j] < Credits + /\ emitted' = [emitted EXCEPT ![j] = @ + 1] + /\ sched' = [sched EXCEPT ![j] = @ + 1] + /\ UNCHANGED <> + +\* A scheduled emit fires: enqueued while the notification is pending (else +\* dropped, its credit returning). The last one takes a held result for +\* release (ReleaseHeld enqueues its terminal message). +FireScheduled(j) == + /\ sched[j] > 0 + /\ sched' = [sched EXCEPT ![j] = @ - 1] + /\ LET fires == Accepting(j) + release == held[j] /\ sched[j] = 1 + IN /\ mbox' = IF fires THEN Append(mbox, [j |-> j, kind |-> "emit"]) ELSE mbox + /\ inflight' = IF fires THEN [inflight EXCEPT ![j] = @ + 1] ELSE inflight + /\ IF release + THEN /\ held' = [held EXCEPT ![j] = FALSE] + /\ releasing' = [releasing EXCEPT ![j] = TRUE] + ELSE UNCHANGED <> + /\ UNCHANGED <> -\* Physical completion; the notification is a one-shot. +\* The released result's terminal message, after the job's lock was +\* released: enqueued unless a deadline or a cancel came in between. The +\* job's credit returns either way. +ReleaseHeld(j) == + /\ releasing[j] + /\ releasing' = [releasing EXCEPT ![j] = FALSE] + /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ IF Accepting(j) + THEN /\ notif' = [notif EXCEPT ![j] = "notified"] + /\ mbox' = Append(mbox, [j |-> j, kind |-> "done"]) + /\ termSent' = [termSent EXCEPT ![j] = @ + 1] + ELSE UNCHANGED <> + /\ UNCHANGED <> + +\* Physical completion; the notification is a one-shot. With emits still +\* scheduled the result is held (keeping the credit); otherwise the terminal +\* message is enqueued, or nothing when the notification was already taken +\* or suppressed, and the credit returns. Finish(j) == /\ exec[j] = "running" /\ exec' = [exec EXCEPT ![j] = "finished"] - /\ IF notif[j] = "pending" \/ (Fault = "DoubleTerminal" /\ notif[j] = "notified") + /\ IF (Accepting(j) /\ sched[j] = 0) + \/ (Fault = "DoubleTerminal" /\ notif[j] = "notified") THEN /\ notif' = [notif EXCEPT ![j] = "notified"] /\ mbox' = Append(mbox, [j |-> j, kind |-> "done"]) /\ termSent' = [termSent EXCEPT ![j] = @ + 1] - ELSE UNCHANGED <> - /\ UNCHANGED <> + /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ UNCHANGED held + ELSE IF Accepting(j) + THEN /\ held' = [held EXCEPT ![j] = TRUE] + /\ credit' = IF Fault = "EarlyCredit" THEN [credit EXCEPT ![j] = FALSE] + ELSE credit + /\ UNCHANGED <> + ELSE /\ credit' = [credit EXCEPT ![j] = FALSE] + /\ UNCHANGED <> + /\ UNCHANGED <> \* Job deadline (only for jobs started with one): the notification is -\* DeadlineExceeded now; the work keeps running and keeps its permit. +\* DeadlineExceeded now. Running work keeps running and keeps its credit; a +\* held result is dropped and its credit returns. JobDeadline(j) == - /\ hasDl[j] /\ exec[j] = "running" /\ notif[j] = "pending" + /\ hasDl[j] /\ notif[j] = "pending" + /\ exec[j] = "running" \/ (held[j] /\ Fault # "HeldDeadlineLost") \/ releasing[j] /\ notif' = [notif EXCEPT ![j] = "notified"] /\ mbox' = Append(mbox, [j |-> j, kind |-> "deadline"]) /\ termSent' = [termSent EXCEPT ![j] = @ + 1] - /\ UNCHANGED <> + /\ held' = [held EXCEPT ![j] = FALSE] + /\ credit' = IF held[j] \/ Fault = "ReleaseOnTimeout" + THEN [credit EXCEPT ![j] = FALSE] ELSE credit + /\ UNCHANGED <> Cancel(j) == /\ alive /\ owner[j] = inst /\ exec[j] # "none" /\ ~cancelled[j] /\ CancelSet({j}) - /\ UNCHANGED <> + /\ UNCHANGED <> -\* Owner exit cancels the live job set. Queued messages stay addressed to -\* the slot (the conservative case); they are fenced at dispatch. +\* Owner exit cancels the live job set (running, or with emits scheduled or +\* a result held). Queued messages stay addressed to the slot (the +\* conservative case); they are fenced at dispatch. A releasing job may or +\* may not still be in the job list the exit cancels (the list is compacted +\* only at size thresholds): either way. Exit == /\ alive /\ alive' = FALSE - /\ CancelSet({j \in Jobs : owner[j] = inst /\ exec[j] = "running"}) - /\ UNCHANGED <> + /\ \E R \in SUBSET {j \in Jobs : owner[j] = inst /\ releasing[j]} : + CancelSet({j \in Jobs : owner[j] = inst + /\ (exec[j] = "running" + \/ ((sched[j] > 0 \/ held[j]) /\ Fault # "ExitKeepsHeld"))} + \cup R) + /\ UNCHANGED <> Restart == /\ ~alive /\ inst < MaxGen /\ inst' = inst + 1 /\ alive' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> Fenced(m) == \/ owner[m.j] # inst /\ Fault # "NoInstanceCheck" @@ -157,14 +281,18 @@ Dispatch == ELSE seen' = seen \cup {[j |-> m.j, kind |-> m.kind, owner |-> owner[m.j], at |-> inst, cancelled |-> cancelled[m.j], superseded |-> m.j \in Keyed /\ Pos(m.j) < Len(started)]} - /\ UNCHANGED <> + /\ UNCHANGED <> Next == \/ Exit \/ Restart \/ Dispatch \/ \E j \in Jobs : Start(j) \/ Emit(j) \/ Finish(j) \/ JobDeadline(j) \/ Cancel(j) + \/ ScheduleEmit(j) \/ FireScheduled(j) \/ ReleaseHeld(j) Spec == Init /\ [][Next]_vars -FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j))) /\ WF_vars(Dispatch) +FairSpec == Spec /\ (\A j \in Jobs : WF_vars(JobDeadline(j)) /\ WF_vars(FireScheduled(j)) + /\ WF_vars(ReleaseHeld(j))) + /\ WF_vars(Dispatch) ------------------------------------------------------------------------------ \* Properties @@ -183,17 +311,30 @@ NoStaleAfterCancel == \A s \in seen : ~s.cancelled NoStaleKeyed == \A s \in seen : ~s.superseded AtMostOneTerminal == \A j \in Jobs : termSent[j] <= 1 /\ Cardinality(TermSeen(j)) <= 1 PermitsBounded == Cardinality(Running) <= JobBound +\* Running work and held results count against the bound: a held result +\* keeps the job's credit until its terminal message. +HeldCounted == Cardinality({j \in Jobs : exec[j] = "running" \/ held[j] \/ releasing[j]}) <= JobBound +\* The job deadline stays armed while a result is held. +DeadlineArmedWhileHeld == + \A j \in Jobs : (held[j] /\ hasDl[j] /\ notif[j] = "pending") => ENABLED JobDeadline(j) OneLivePerKey == Cardinality(LiveKeyed) <= 1 \* A job's terminal message is the last message it puts in the mailbox. TerminalLast == \A a, b \in 1..Len(mbox) : (a < b /\ mbox[a].j = mbox[b].j /\ IsTerm(mbox[a])) => FALSE -EmitCreditsBounded == \A j \in Jobs : inflight[j] <= Credits +EmitCreditsBounded == \A j \in Jobs : inflight[j] + sched[j] <= Credits +\* After the owner's exit nothing of its instance is live: running work, +\* scheduled emits and held results are all cancelled (their notification +\* suppressed, held results dropped). +ExitCancelsJobs == + ~alive => \A j \in Jobs : + (owner[j] = inst /\ (exec[j] = "running" \/ sched[j] > 0 \/ held[j])) => cancelled[j] Safety == /\ TypeOK /\ NoLateResult /\ NoStaleAfterCancel /\ NoStaleKeyed /\ AtMostOneTerminal /\ PermitsBounded /\ OneLivePerKey - /\ TerminalLast /\ EmitCreditsBounded + /\ TerminalLast /\ EmitCreditsBounded /\ HeldCounted /\ DeadlineArmedWhileHeld + /\ ExitCancelsJobs \* A job started with a deadline and never cancelled is reported to its \* live owner instance even if its work never finishes. diff --git a/verification/tla/DeferredReplies.tla b/verification/tla/DeferredReplies.tla index 7207f24cf..2faa5324d 100644 --- a/verification/tla/DeferredReplies.tla +++ b/verification/tla/DeferredReplies.tla @@ -5,10 +5,13 @@ (* Part 2 (job fencing, keyed replacement, emits) is DeferredJobs.tla. *) (* *) (* Each request owns a slot with one atomic open -> terminal transition. *) -(* The deadline is fixed at send and is the only timeout; a reply at or *) -(* after it expires the slot instead of delivering. Expiry is serviced *) -(* independently of A's turns ("waiting" callers are polling; deferred *) -(* tokens are covered by the deadline service). *) +(* The deadline is fixed at send and is the only timeout, and only the *) +(* expiry (Expire: the caller's timer, the owner's deadline heap, or a *) +(* [deadline] check) decides it, as OTP's gen_server:call: a reply reads *) +(* no clock and delivers while the slot is open; one that loses the race *) +(* with the expiry finds the slot closed and its value is discarded. *) +(* Expiry is serviced independently of A's turns ("waiting" callers are *) +(* polling; deferred tokens are covered by the deadline service). *) (* *) (* Owner A. Queued requests are dispatched even if their slot is already *) (* terminal (OTP: the server still handles the call). In a turn the *) @@ -16,7 +19,7 @@ (* Overloaded at BoundA, or nothing if the slot is already closed) into a *) (* local token, then stores it, pipes it into a linked job, or drops it; *) (* drops in a turn commit when the turn ends: Dropped if it returns, *) -(* Stopped if it panics. A linked job shares its token's credit while the *) +(* exit pending if it panics. A linked job shares its token's credit while the *) (* token is open and no other job of that request runs; otherwise it *) (* needs a fresh credit. A job's notification hands the token back to a *) (* turn (completion or job deadline; after a deadline the work keeps *) @@ -26,9 +29,22 @@ (* *) (* Exit: A enters "exiting" (stop between turns, or a panic in a turn). *) (* While exiting (stopped(), OTP terminate) stored tokens can still be *) -(* answered, drops answer Stopped, nothing is dispatched and jobs are *) -(* cancelled. Death closes the registry (every token A holds, wherever it *) -(* is) and discards the mailbox. If stopped() (or a destructor) is still *) +(* answered, nothing is dispatched and jobs are cancelled. A token *) +(* dropped while exiting, or unwound by the panicking turn, is "exit *) +(* pending" (pendA): the registry keeps it open and death answers it with *) +(* the final reason (docs/design/otp-exit-signals.md, "Required changes *) +(* to rank 11"); only the caller's deadline or abandonment comes first. *) +(* Death follows actor_cleanup.rs. Death (the point D) fences the reply *) +(* gate (a reply already past it has discharged; none is admitted after) *) +(* and seals the registry ("sealed": no new request is admitted); callers,*) +(* the deadline service and stored replies race it. Answer then closes *) +(* every token A holds (wherever it is), every exit-pending slot and the *) +(* mailbox with Stopped, as one batch of terminal transitions, and runs *) +(* the collected wake-ups after the last answer ("dead"): no waker, *) +(* callback or destructor runs in between. The model has no *) +(* reasons: the reason is fixed at death (Lifecycle.tla), so StoppedAtDeath*) +(* (Stopped only in the death step) is what makes callers see it. *) +(* If stopped() (or a destructor) is still *) (* running at the cleanup deadline, A becomes "stuck": nothing is closed *) (* early (ActorStopped means the actor is gone); stopped() may still *) (* reply; callers are bounded by their own deadlines; death may never come.*) @@ -37,7 +53,8 @@ (* violate the named invariant: *) (* NoAutoReply (NoOrphan) drops leave the slot open *) (* NoAtomicExpiry (AtMostOneDischarge) reply after expiry discharges *) -(* LateSuccess (NoSuccessAfterDeadline) *) +(* LateSuccess (NoSuccessAfterDeadline) a reply overrides an expired *) +(* slot (delivered after expiry) *) (* LoopDeadline (TimelyExpiry) expiry waits for A's turn *) (* Lengthen (DeadlineWithinCaller) *) (* Unbounded (BoundedA) defer ignores the bound *) @@ -49,31 +66,40 @@ (* LeakDownstream (DownstreamOwned) B's request outlives its job *) (* EarlyClose (NoEarlyClose) the cleanup deadline closes the *) (* registry before death *) +(* EarlyStopped (StoppedAtDeath) exit errors answered Stopped at *) +(* once (rank 11 before 5e) *) +(* NoReplyFence (NoReplyAfterDeath) a reply is admitted after D *) +(* CallbackBeforeDischarge (WokenAfterDischarge) an answer's wake-up *) +(* runs before the last answer *) (***************************************************************************) EXTENDS Naturals, FiniteSets CONSTANTS N, BoundA, BoundB, MaxTime, MaxWait, Fault ASSUME Fault \in {"None", "NoAutoReply", "NoAtomicExpiry", "LateSuccess", "LoopDeadline", "Lengthen", "Unbounded", "NoDestAdmission", "ExitStateOnly", "UnwindDropped", "ShareRunning", - "AttachClosed", "LeakDownstream", "EarlyClose"} + "AttachClosed", "LeakDownstream", "EarlyClose", + "EarlyStopped", "NoReplyFence", "CallbackBeforeDischarge"} Req == 1..N \* qA queued; turnA PendingReply in a turn; localA ReplyTo in a turn; \* dropA dropped in a turn (commits at turn end); stateA stored; -\* doneA captured by a linked job's done; gone no longer held. -Holders == {"none", "qA", "turnA", "localA", "dropA", "stateA", "doneA", "gone"} +\* doneA captured by a linked job's done; pendA exit pending (dropped while +\* exiting or unwound; the registry answers it at death); gone no longer held. +Holders == {"none", "qA", "turnA", "localA", "dropA", "stateA", "doneA", "pendA", "gone"} Terminal == {"ok", "timeout", "overloaded", "stopped", "dropped", "abandoned"} VARIABLES now, caller, sent, dl, slot, holder, discharges, okAt, jobs, \* running linked jobs per request (0..2) down, \* downstream request to B: none, qB, stateB, done, abandoned - phase, \* A: run, exiting, stuck (cleanup deadline passed), dead + phase, \* A: run, exiting, stuck (cleanup deadline passed), + \* sealed (past D, answering), dead (answered, woken) turnA, \* request in A's turn, 0 if idle panicked, \* history: requests whose token was in a panicking turn - earlyClosed \* history: tokens answered Stopped by a close before death + earlyClosed, \* history: tokens answered Stopped by a close before death + lateOk \* history: requests answered Ok by a reply admitted after D vars == <> + phase, turnA, panicked, earlyClosed, lateOk>> Init == /\ now = 0 /\ caller = [r \in Req |-> "idle"] @@ -81,12 +107,14 @@ Init == /\ slot = [r \in Req |-> "none"] /\ holder = [r \in Req |-> "none"] /\ discharges = [r \in Req |-> 0] /\ okAt = [r \in Req |-> 0] /\ jobs = [r \in Req |-> 0] /\ down = [r \in Req |-> "none"] - /\ phase = "run" /\ turnA = 0 /\ panicked = {} /\ earlyClosed = {} + /\ phase = "run" /\ turnA = 0 /\ panicked = {} /\ earlyClosed = {} /\ lateOk = {} Open(r) == slot[r] = "open" TokenHeld(r) == Open(r) /\ holder[r] \in {"localA", "dropA", "stateA", "doneA"} \* Credits: a request's token and its first job share one credit; every \* further running job, and every job whose token has closed, needs its own. +\* An exit-pending token keeps its credit until death in the implementation; +\* nothing takes a credit once exit has begun, so it is not counted here. CreditOf(r) == IF jobs[r] > 0 THEN jobs[r] ELSE IF TokenHeld(r) THEN 1 ELSE 0 Credits == Cardinality({<> \in Req \X {1, 2} : k <= CreditOf(r)}) HeldB == {r \in Req : down[r] = "stateB"} @@ -101,20 +129,29 @@ Discharge(r, k) == DischargeSet({r}, k) Keep == UNCHANGED <> OpenOf(S) == {r \in S : Open(r)} +\* A reply reads no clock (OTP): it delivers while the slot is open; after +\* the expiry it finds the slot closed and its value is discarded. TryReply(r) == IF Open(r) - THEN IF now < dl[r] \/ Fault = "LateSuccess" THEN Discharge(r, "ok") - ELSE Discharge(r, "timeout") + THEN Discharge(r, "ok") + ELSE IF Fault = "LateSuccess" /\ slot[r] = "timeout" + THEN /\ slot' = [slot EXCEPT ![r] = "ok"] + /\ okAt' = [okAt EXCEPT ![r] = now] + /\ UNCHANGED <> ELSE IF Fault = "NoAtomicExpiry" /\ slot[r] = "timeout" THEN /\ discharges' = [discharges EXCEPT ![r] = @ + 1] /\ okAt' = [okAt EXCEPT ![r] = now] /\ UNCHANGED <> ELSE Keep -\* A drop outside a turn: Dropped while running, Stopped while exiting. +\* A drop outside a turn: Dropped while running; exit pending while exiting +\* (answered at death with the final reason). DropNow(r) == - IF ~Open(r) \/ Fault = "NoAutoReply" THEN Keep - ELSE Discharge(r, IF phase = "run" THEN "dropped" ELSE "stopped") + IF ~Open(r) \/ Fault = "NoAutoReply" THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Keep + ELSE IF phase = "run" THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "dropped") + ELSE IF Fault = "EarlyStopped" + THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "stopped") + ELSE holder' = [holder EXCEPT ![r] = "pendA"] /\ Keep U(v) == UNCHANGED v @@ -122,7 +159,7 @@ U(v) == UNCHANGED v \* Callers Send(r) == - /\ caller[r] = "idle" /\ phase # "dead" + /\ caller[r] = "idle" /\ phase \notin {"sealed", "dead"} /\ \E d \in 1..MaxWait : now + d <= MaxTime /\ sent' = [sent EXCEPT ![r] = now + d] /\ dl' = [dl EXCEPT ![r] = now + d] /\ slot' = [slot EXCEPT ![r] = "open"] /\ holder' = [holder EXCEPT ![r] = "qA"] @@ -201,38 +238,47 @@ EndTurn == /\ turnA' = 0 /\ U(<>) -\* Panic in a turn: the turn's tokens are dropped while unwinding and -\* answer Stopped; exit begins. +\* Panic in a turn: the turn's tokens are dropped while unwinding and are +\* exit pending (answered at death with the final reason); exit begins. Panic == /\ phase = "run" /\ turnA # 0 /\ LET r == turnA IN /\ panicked' = panicked \cup (IF holder[r] \in {"turnA", "localA", "dropA"} /\ Open(r) THEN {r} ELSE {}) - /\ IF holder[r] \in {"turnA", "localA", "dropA"} - THEN holder' = [holder EXCEPT ![r] = "gone"] - /\ IF Open(r) THEN Discharge(r, IF Fault = "UnwindDropped" THEN "dropped" - ELSE "stopped") - ELSE Keep + /\ IF holder[r] \in {"turnA", "localA", "dropA"} /\ Open(r) + THEN (CASE Fault = "UnwindDropped" -> + holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "dropped") + [] Fault = "EarlyStopped" -> + holder' = [holder EXCEPT ![r] = "gone"] /\ Discharge(r, "stopped") + [] OTHER -> holder' = [holder EXCEPT ![r] = "pendA"] /\ Keep) + ELSE IF holder[r] \in {"turnA", "localA", "dropA"} + THEN holder' = [holder EXCEPT ![r] = "gone"] /\ Keep ELSE U(holder) /\ Keep /\ phase' = "exiting" /\ turnA' = 0 /\ U(<>) \* A later turn, or stopped() while exiting, answers or drops a stored token. +\* The reply gate admits it only before D (the reply permit; Death waits +\* out a reply already admitted). +ReplyGateOpen == phase \in {"run", "exiting", "stuck"} + \/ (Fault = "NoReplyFence" /\ phase = "sealed") LaterReply(r) == - /\ phase \in {"run", "exiting", "stuck"} /\ turnA = 0 /\ holder[r] = "stateA" + /\ ReplyGateOpen /\ turnA = 0 /\ holder[r] = "stateA" /\ holder' = [holder EXCEPT ![r] = "gone"] /\ TryReply(r) + /\ lateOk' = IF phase = "sealed" /\ Open(r) THEN lateOk \cup {r} ELSE lateOk /\ U(<>) LaterDrop(r) == /\ phase \in {"run", "exiting", "stuck"} /\ turnA = 0 /\ holder[r] = "stateA" - /\ holder' = [holder EXCEPT ![r] = "gone"] /\ DropNow(r) + /\ DropNow(r) /\ U(<>) ExitBegin == /\ phase = "run" /\ turnA = 0 /\ phase' = "exiting" /\ U(<>) -\* Death: the registry closes every token A holds; the mailbox is discarded. +\* Death: the registry closes every token A holds and every exit-pending +\* slot, with the final reason; the mailbox is discarded. \* Cleanup deadline while stopped() still runs: the instance is reported \* stuck; nothing is closed early, since ActorStopped means "gone". CleanupDeadline == @@ -243,12 +289,49 @@ CleanupDeadline == ELSE Keep /\ UNCHANGED earlyClosed /\ U(<>) +\* The death point D: the reply gate is fenced and the registry sealed. Death == - /\ phase \in {"exiting", "stuck"} /\ phase' = "dead" - /\ DischargeSet(OpenOf({r \in Req : holder[r] \in - (IF Fault = "ExitStateOnly" THEN {"qA", "stateA"} - ELSE {"qA", "stateA", "doneA"})}), "stopped") - /\ U(<>) + /\ phase \in {"exiting", "stuck"} /\ phase' = "sealed" + /\ Keep /\ U(<>) + +\* What death answers: every token A holds and every exit-pending slot. +DeathHeld == OpenOf({r \in Req : holder[r] \in + (IF Fault = "ExitStateOnly" THEN {"qA", "stateA", "pendA"} + ELSE {"qA", "stateA", "doneA", "pendA"})}) + +\* The batch of answers and then its collected wake-ups. Each slot's +\* transition is its own CAS, and the only other steps on a held slot +\* (expiry, abandonment) are terminal transitions of that slot, so an +\* interleaving of them with the batch reaches the same states as one taken +\* before it; the wake-ups follow the batch with no user code in between. +\* So the batch and its wake-ups are one step (Answer); what the model keeps +\* apart is D (Death) and the batch, with callers, the deadline service and +\* replies racing D in between. CallbackBeforeDischarge answers one slot at +\* a time, waking its caller at once, and ends with Discharged. +Answer(r) == + /\ phase = "sealed" + /\ IF Fault = "CallbackBeforeDischarge" + THEN /\ r \in DeathHeld + /\ slot' = [slot EXCEPT ![r] = "stopped"] + /\ discharges' = [discharges EXCEPT ![r] = @ + 1] + /\ caller' = IF caller[r] = "waiting" THEN [caller EXCEPT ![r] = "err"] ELSE caller + /\ UNCHANGED phase + ELSE /\ r = 1 + /\ slot' = [q \in Req |-> IF q \in DeathHeld THEN "stopped" ELSE slot[q]] + /\ discharges' = [q \in Req |-> IF q \in DeathHeld THEN discharges[q] + 1 + ELSE discharges[q]] + /\ caller' = [q \in Req |-> IF q \in DeathHeld /\ caller[q] = "waiting" + THEN "err" ELSE caller[q]] + /\ phase' = "dead" + /\ U(<>) + +\* The batch is done: the collected wake-ups run. +Discharged == + /\ Fault = "CallbackBeforeDischarge" + /\ phase = "sealed" /\ DeathHeld = {} /\ phase' = "dead" + /\ caller' = [r \in Req |-> IF slot[r] = "stopped" /\ caller[r] = "waiting" + THEN "err" ELSE caller[r]] + /\ U(<>) ------------------------------------------------------------------------------ \* Linked jobs and the downstream actor B @@ -279,7 +362,7 @@ JobEnds(r) == JobCancel(r) == /\ holder[r] = "doneA" /\ (~Open(r) \/ phase = "exiting") - /\ holder' = [holder EXCEPT ![r] = "gone"] /\ DropNow(r) + /\ DropNow(r) /\ U(<>) JobCallB(r) == @@ -322,21 +405,26 @@ Tick == ------------------------------------------------------------------------------ NextOther == - \/ Tick \/ EndTurn \/ Panic \/ ExitBegin \/ Death + \/ Tick \/ EndTurn \/ Panic \/ ExitBegin \/ Death \/ Discharged \/ \E r \in Req : + \/ Answer(r) \/ Send(r) \/ CallerGivesUp(r) \/ DispatchA(r) \/ ReplyNow(r) \/ Defer(r) - \/ Store(r) \/ DropInTurn(r) \/ PipeReply(r) \/ LaterReply(r) \/ LaterDrop(r) + \/ Store(r) \/ DropInTurn(r) \/ PipeReply(r) \/ LaterDrop(r) \/ JobNotify(r) \/ JobEnds(r) \/ JobCancel(r) \/ JobCallB(r) \/ BDispatch(r) \/ BReply(r) \/ Shorten(r) \/ Expire(r) Next == - \/ CleanupDeadline - \/ (UNCHANGED earlyClosed /\ NextOther) + \/ CleanupDeadline /\ UNCHANGED lateOk + \/ \E r \in Req : LaterReply(r) /\ UNCHANGED earlyClosed + \/ (UNCHANGED <> /\ NextOther) Spec == Init /\ [][Next]_vars -\* Fairness only for the clock and the deadline service. -FairSpec == Spec /\ WF_vars(Tick /\ UNCHANGED earlyClosed) - /\ \A r \in Req : WF_vars(Expire(r) /\ UNCHANGED earlyClosed) +\* Fairness for the clock, the deadline service and the steps of death +\* after D (framework code: no user code runs between them). +Quiet == UNCHANGED <> +FairSpec == Spec /\ WF_vars(Tick /\ Quiet) + /\ \A r \in Req : WF_vars(Expire(r) /\ Quiet) /\ WF_vars(Answer(r) /\ Quiet) + /\ WF_vars(Discharged /\ Quiet) ------------------------------------------------------------------------------ \* Properties @@ -344,31 +432,48 @@ FairSpec == Spec /\ WF_vars(Tick /\ UNCHANGED earlyClosed) TypeOK == /\ now \in 0..MaxTime /\ holder \in [Req -> Holders] /\ slot \in [Req -> {"none", "open"} \cup Terminal] - /\ jobs \in [Req -> 0..2] /\ phase \in {"run", "exiting", "stuck", "dead"} + /\ jobs \in [Req -> 0..2] /\ phase \in {"run", "exiting", "stuck", "sealed", "dead"} AtMostOneDischarge == \A r \in Req : discharges[r] <= 1 NoOrphan == \A r \in Req : Open(r) => /\ phase # "dead" - /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA"} -NoSuccessAfterDeadline == \A r \in Req : slot[r] = "ok" => okAt[r] < dl[r] + /\ holder[r] \in {"qA", "turnA", "localA", "dropA", "stateA", "doneA", "pendA"} +\* A reply succeeds only while its slot is open: never after the expiry, so +\* no later than the deadline (the clock does not pass an open slot's +\* deadline: TimelyExpiry). It may land at the deadline instant, before the +\* expiry runs, as an OTP reply already in the caller's mailbox does. +NoSuccessAfterDeadline == \A r \in Req : slot[r] = "ok" => okAt[r] <= dl[r] TimelyExpiry == \A r \in Req : Open(r) => now <= dl[r] DeadlineWithinCaller == \A r \in Req : slot[r] # "none" => dl[r] <= sent[r] BoundedA == Credits <= BoundA BoundedB == Cardinality(HeldB) <= BoundB -PanicStopped == \A r \in panicked : slot[r] = "stopped" +\* A request open in a panicking turn is never answered Dropped (or a value): +\* it waits for death (Stopped), unless its deadline or caller comes first. +PanicStopped == \A r \in panicked : slot[r] \in {"open", "stopped", "timeout", "abandoned"} +\* Exit errors wait for death: a Stopped discharge happens only in the death +\* step, where the reason is final (otp-exit-signals.md; the same property +\* Lifecycle.tla checks as StoppedAtDeath). +StoppedAtDeath == \A r \in Req : slot[r] = "stopped" => phase \in {"sealed", "dead"} +\* No reply is admitted after D: an Ok always comes from a live instance. +NoReplyAfterDeath == lateOk = {} +\* A caller woken with Stopped finds the whole death answered: no answer's +\* wake-up (user code) runs while another request A held is still open. +WokenAfterDischarge == + (\E r \in Req : slot[r] = "stopped" /\ caller[r] = "err") => DeathHeld = {} \* The registry closes only at actual death: no token is answered Stopped \* by a close while the instance (its stopped()) is still alive. NoEarlyClose == earlyClosed = {} DownstreamOwned == \A r \in Req : down[r] \in {"qB", "stateB"} => jobs[r] > 0 CallerConsistent == - \A r \in Req : /\ caller[r] = "ok" => (slot[r] = "ok" /\ okAt[r] < sent[r]) + \A r \in Req : /\ caller[r] = "ok" => (slot[r] = "ok" /\ okAt[r] <= sent[r]) /\ caller[r] = "timeout" => slot[r] = "timeout" Safety == /\ TypeOK /\ AtMostOneDischarge /\ NoOrphan /\ NoSuccessAfterDeadline /\ TimelyExpiry /\ DeadlineWithinCaller /\ BoundedA /\ BoundedB - /\ PanicStopped /\ NoEarlyClose /\ DownstreamOwned /\ CallerConsistent + /\ PanicStopped /\ StoppedAtDeath /\ NoEarlyClose /\ DownstreamOwned + /\ CallerConsistent /\ NoReplyAfterDeath /\ WokenAfterDischarge EveryAcceptedDischarged == \A r \in Req : (slot[r] = "open") ~> (slot[r] \in Terminal) CallerNeverHangs == \A r \in Req : (caller[r] = "waiting") ~> (caller[r] # "waiting") diff --git a/verification/tla/Lifecycle.tla b/verification/tla/Lifecycle.tla index 1e5b99348..7e3863f3e 100644 --- a/verification/tla/Lifecycle.tla +++ b/verification/tla/Lifecycle.tla @@ -10,10 +10,11 @@ EXTENDS Naturals, Sequences, FiniteSets (* "run" dispatching; *) (* "stopping" exit begun: no dispatch; stopped() may run and send; an *) (* untrappable kill still replaces the requested reason; *) -(* "gone" past the death point D: the reason is final, the mailbox *) -(* refuses admission, every queued request was answered, the *) +(* "gone" past the death point D: the reason is final, outgoing exit *) +(* signals are fenced, the mailbox refuses admission, the *) (* link and monitor table is closed and one notice per edge *) -(* was taken; destruction (state drop) is in progress; *) +(* was taken; then (Discharge) every queued request is *) +(* answered, and destruction (state drop) follows; *) (* "dead" published: wait_exit/join return; the notices taken at D *) (* may now be delivered. *) (* Registration (link, monitor) and request admission are atomic with D: *) @@ -27,6 +28,10 @@ EXTENDS Naturals, Sequences, FiniteSets (* termination when dispatched; a Normal signal is ignored by a *) (* non-trapping actor; any other reason begins exit with that reason. *) (* Administrative stops (ChildHandle::stop and friends) are untrappable. *) +(* Not modelled: phase 0 (signals arriving while started() runs are held *) +(* as envelopes in arrival order and resolved with the trap flag read when *) +(* it returns, link.rs ExitRequest::end_starting); actors start in "run". *) +(* Integration tests cover it (otp_exit_signals, signals during started). *) (* Handlers are atomic steps; "after the current handler" is the next *) (* step. Liveness assumes handlers, stopped() and destruction return *) (* (weak fairness on Dispatch, Die and Publish). *) @@ -68,11 +73,15 @@ Past(a, s) == s \in {"gone", "dead"} (* SplitDeathPoint registration checks publication instead of D *) (* NoUnlinkFence unlink does not cancel a link notice in flight *) (* NoDemonitorFence demonitor does not cancel a Down in flight *) +(* NoDemonitorFlush a Down already queued runs after demonitor (v0.7) *) (* EarlyStopped a panicking request is answered before D *) (* PanicOverKill a panic during exit replaces Killed *) (* DropKeepsNormal a runtime drop keeps a requested Normal *) (* AlwaysHook v0.7: stopped() runs after a kill or untrapped signal *) (* LateSignalIgnored v0.7: a late signal during exit keeps the reason *) +(* NoSignalFence an exit signal sent by an actor takes effect after D *) +(* PublishBeforeDischarge publication (join, notices) before every queued *) +(* request was answered *) KilledReason == IF Fault = "KillAsKill" THEN "Kill" ELSE "Killed" VARIABLES @@ -96,16 +105,18 @@ VARIABLES hook, \* [Actors -> BOOLEAN]: stopped() runs (ghost) began, \* [Actors -> causes]: what began the exit (ghost) lateSig, \* [Actors -> reason of the last late signal applied, or None] (ghost) + disch, \* [Actors -> BOOLEAN]: past D, the queued requests were answered + sigAfterD, \* an exit signal took effect after its sender's D (ghost) sends, signals, lates, psent, \* children that received the supervisor's Shutdown afterStop \* a message was dispatched after stop was observed (ghost) vars == <> -life == <> + lates, psent, afterStop, disch, sigAfterD>> +life == <> reg == <> -budget == <> +budget == <> \* Mailbox items. o: "death" (link or monitor notice taken at D), "late" \* (registration after D), "direct" (an exit signal sent by a live sender), @@ -142,6 +153,7 @@ Init == /\ hook = [a \in Actors |-> FALSE] /\ began = [a \in Actors |-> None] /\ lateSig = [a \in Actors |-> None] + /\ disch = [a \in Actors |-> FALSE] /\ sigAfterD = FALSE /\ sends = 0 /\ signals = 0 /\ lates = 0 /\ psent = {} /\ afterStop = FALSE @@ -158,7 +170,7 @@ Begin(a, r, c) == \* skips terminate). /\ hook' = [hook EXCEPT ![a] = c \in {"self", "admin", "parent"} \/ Fault = "AlwaysHook"] /\ began' = [began EXCEPT ![a] = c] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE UNCHANGED life (* An untrappable kill: begins exit, or replaces the requested reason of an *) @@ -172,14 +184,14 @@ Kill(a) == /\ killed' = [killed EXCEPT ![a] = TRUE] /\ hook' = [hook EXCEPT ![a] = Fault = "AlwaysHook"] /\ began' = [began EXCEPT ![a] = "kill"] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE IF st[a] = "stopping" THEN /\ why' = IF Fault = "KillNoSupersede" THEN why ELSE [why EXCEPT ![a] = KilledReason] /\ cause' = [cause EXCEPT ![a] = "kill"] /\ killed' = [killed EXCEPT ![a] = TRUE] /\ lateSig' = [lateSig EXCEPT ![a] = None] \* a stopped() that already began runs to its end (R) - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE UNCHANGED life (* An exit signal from s with reason r and origin o arrives at a. c is the *) @@ -192,7 +204,7 @@ Signal(a, s, r, o, c) == THEN /\ why' = IF Fault = "LateSignalIgnored" THEN why ELSE [why EXCEPT ![a] = r] /\ cause' = IF Fault = "LateSignalIgnored" THEN cause ELSE [cause EXCEPT ![a] = c] /\ lateSig' = [lateSig EXCEPT ![a] = r] - /\ UNCHANGED <> + /\ UNCHANGED <> ELSE IF st[a] # "run" THEN UNCHANGED <> ELSE IF trap[a] THEN /\ Push(a, Item(IF IsParent(a, s) /\ Fault # "ParentAsExit" THEN "pexit" ELSE "exit", @@ -216,15 +228,33 @@ Fire(a, r) == ELSE mstate[m]] /\ fired' = fired \cup {a} -(* The death point D of a with final reason r: admission closes, queued *) -(* requests are answered, the table closes. Shared by Die and RuntimeDrop. *) +(* The death point D of a with final reason r, as in actor_cleanup.rs: *) +(* outgoing exit signals (and replies) are fenced, admission closes and *) +(* the table closes. Shared by Die and RuntimeDrop. The queued and exit- *) +(* pending requests are answered by the separate Discharge step, which *) +(* other actors' steps may precede; publication waits for it. (Deferred *) +(* replies and the wake-up batch are modelled step by step in *) +(* DeferredReplies.tla.) With no request to answer, Discharge only drops *) +(* the mailbox, which nothing reads past D, so it is folded into Death. *) +Pending(a) == \E q \in Reqs : req[q].s \in {"queued", "exiting"} /\ req[q].t = a Death(a, r) == /\ st' = [st EXCEPT ![a] = "gone"] + /\ IF Pending(a) THEN UNCHANGED <> + ELSE /\ mbox' = [mbox EXCEPT ![a] = <<>>] + /\ disch' = [disch EXCEPT ![a] = TRUE] + /\ IF a \in fired THEN UNCHANGED <> ELSE Fire(a, r) + +(* Past D: every queued or exit-pending request is answered ActorStopped *) +(* with the final reason; the mailbox is disposed of. *) +Discharge(a) == + /\ st[a] = "gone" /\ ~disch[a] + /\ disch' = [disch EXCEPT ![a] = TRUE] /\ mbox' = [mbox EXCEPT ![a] = <<>>] /\ req' = [q \in Reqs |-> IF req[q].s \in {"queued", "exiting"} /\ req[q].t = a /\ Fault # "DropQueued" - THEN [req[q] EXCEPT !.s = "stopped", !.why = r] ELSE req[q]] - /\ IF a \in fired THEN UNCHANGED <> ELSE Fire(a, r) + THEN [req[q] EXCEPT !.s = "stopped", !.why = why[a]] ELSE req[q]] + /\ UNCHANGED <> ---------------------------------------------------------------------------- (* Turns of a running actor. *) @@ -239,7 +269,7 @@ Send(a, b) == \* also from stopped(), when it runs /\ a # b /\ st[b] \in {"run", "stopping"} /\ sends < MaxSends /\ sends' = sends + 1 /\ Push(b, Item("msg", a, None, 0, 0, "none")) - /\ UNCHANGED <> + /\ UNCHANGED <> SelfExit(a, r) == \* ctx.stop, ctx.stop_with_reason(r), a panic in any turn /\ st[a] = "run" /\ r \in SelfReasons @@ -255,7 +285,7 @@ PanicWhileExiting(a) == /\ why' = [why EXCEPT ![a] = "Panic"] /\ cause' = [cause EXCEPT ![a] = "self"] /\ lateSig' = [lateSig EXCEPT ![a] = None] - /\ UNCHANGED <> + /\ UNCHANGED <> Link(a, b) == \* idempotent while a link or a pending link notice exists /\ st[a] = "run" /\ a # b /\ {a, b} \notin links @@ -273,7 +303,7 @@ Link(a, b) == \* idempotent while a link or a pending link notice exists ELSE /\ links' = links \cup {{a, b}} /\ UNCHANGED <> /\ UNCHANGED <> + req, sends, signals, psent, afterStop, sigAfterD>> (* Unlink: removes the link, and cancels a notice from b already taken at *) (* b's death but not yet delivered (OTP: once unlink returns, the link has *) @@ -283,7 +313,7 @@ Link(a, b) == \* idempotent while a link or a pending link notice exists TryLink(a, b) == /\ st[a] = "run" /\ a # b /\ IsGone(b) /\ lates < MaxLate /\ lates' = lates + 1 - /\ UNCHANGED <> + /\ UNCHANGED <> Unlink(a, b) == LET pending == {n \in inflight : n.to = a /\ n.from = b /\ n.k = "link"} @@ -306,13 +336,17 @@ Monitor(a, b, m) == ELSE /\ mstate' = [mstate EXCEPT ![m] = "active"] /\ UNCHANGED <> /\ UNCHANGED <> + req, sends, signals, psent, afterStop, sigAfterD>> (* Demonitor: no Down is enqueued after it returns; one already queued stays. *) +(* With OTP's flush (demonitor(Ref, [flush])), also while the monitor's Down *) +(* is queued in the observer's mailbox: its dispatch then has no effect. *) Demonitor(a, m) == LET pending == {n \in inflight : n.k = "down" /\ n.m = m} + queued == \E i \in 1..Len(mbox[a]) : mbox[a][i].k = "down" /\ mbox[a][i].m = m IN /\ st[a] = "run" /\ mon[m].o = a - /\ mstate[m] \in {"active", "fired", "late"} /\ (mstate[m] = "active" \/ pending # {}) + /\ mstate[m] \in {"active", "fired", "late"} + /\ (mstate[m] = "active" \/ pending # {} \/ queued) /\ mstate' = [mstate EXCEPT ![m] = "cancelled"] /\ cancelled' = cancelled \cup pending /\ inflight' = IF Fault = "NoDemonitorFence" THEN inflight ELSE inflight \ pending @@ -342,14 +376,19 @@ Dispatch(a) == /\ UNCHANGED req [] OTHER -> \* msg or down (a panic here is SelfExit) UNCHANGED <> - /\ UNCHANGED <> + \* A flushed Down (its monitor cancelled by demonitor) is skipped: its + \* handler takes no effect. NoDemonitorFlush runs it anyway. + /\ LET h == Head(mbox[a]) IN + afterCancel' = (afterCancel \/ (h.k = "down" /\ Fault = "NoDemonitorFlush" + /\ mstate[h.m] = "cancelled")) + /\ UNCHANGED <> (* stopped() returned: the death point D. *) Die(a) == /\ st[a] = "stopping" /\ Death(a, why[a]) /\ UNCHANGED <> + budget, afterStop, req>> (* The runtime drops the actor (tasks: the future is dropped; threads: the *) (* closure never ran). No stopped(); reason Killed unless a non-Normal one *) @@ -363,7 +402,8 @@ RuntimeDrop(a) == /\ cause' = [cause EXCEPT ![a] = IF why[a] \in {None, "Normal"} THEN "drop" ELSE cause[a]] /\ trapAt' = [trapAt EXCEPT ![a] = IF st[a] = "run" THEN trap[a] ELSE trapAt[a]] /\ Death(a, r) - /\ UNCHANGED <> + /\ UNCHANGED <> EarlyFire(a) == \* Fault EarlyNotify only /\ Fault = "EarlyNotify" /\ st[a] = "stopping" /\ a \notin fired @@ -372,9 +412,10 @@ EarlyFire(a) == \* Fault EarlyNotify only (* Destruction finished: the exit is published. *) Publish(a) == - /\ st[a] = "gone" + /\ st[a] = "gone" /\ (disch[a] \/ Fault = "PublishBeforeDischarge") /\ st' = [st EXCEPT ![a] = "dead"] - /\ UNCHANGED <> + /\ UNCHANGED <> (* A notice taken at D is delivered after publication; late notices at once. *) Deliver(n) == @@ -414,7 +455,11 @@ Enqueue(r) == (* ChildHandle::exit(r) from outside (s = Ext). r = Kill is kill(). *) ExitSignal(s, a, r) == /\ r \in SignalReasons /\ signals < MaxSignals /\ signals' = signals + 1 - /\ IF s = Ext THEN TRUE ELSE st[s] \in {"run", "stopping"} \* inert after D + \* The signal gate: inert after D (D waits out a signal already past it). + /\ IF s = Ext THEN TRUE + ELSE st[s] \in {"run", "stopping"} + \/ (Fault = "NoSignalFence" /\ st[s] = "gone" /\ ~disch[s]) + /\ sigAfterD' = (sigAfterD \/ (s # Ext /\ Past(s, st[s]))) /\ IF r = "Kill" THEN /\ Kill(a) /\ UNCHANGED mbox ELSE IF Fault = "DirectStop" @@ -425,13 +470,13 @@ ExitSignal(s, a, r) == AdminStop(a, r) == \* ChildHandle::stop / stop_with_reason / shutdown /\ r \in AdminReasons /\ signals < MaxSignals /\ signals' = signals + 1 /\ Begin(a, r, "admin") - /\ UNCHANGED <> + /\ UNCHANGED <> SupShutdown(c) == \* exit(Child, shutdown) from the parent, also to a child /\ c \in Children /\ c \notin psent /\ st[c] \in {"run", "stopping"} \* already exiting /\ psent' = psent \cup {c} /\ Signal(c, Sup, "Shutdown", "direct", "psignal") - /\ UNCHANGED <> + /\ UNCHANGED <> SupKill(c) == \* shutdown_timeout passed: exit(Child, kill) /\ c \in psent /\ st[c] \in {"run", "stopping"} @@ -448,6 +493,7 @@ Next == \/ \E r \in Reasons : SelfExit(a, r) \/ AdminStop(a, r) \/ \E s \in Actors \cup {Ext} : ExitSignal(s, a, r) \/ Dispatch(a) \/ Die(a) \/ RuntimeDrop(a) \/ EarlyFire(a) \/ Publish(a) + \/ Discharge(a) \/ SupShutdown(a) \/ SupKill(a) \/ \E n \in inflight : Deliver(n) \/ \E r \in Reqs : Enqueue(r) \/ \E t \in Actors : Request(r, t) @@ -455,6 +501,7 @@ Next == Fairness == /\ \A a \in Actors : WF_vars(Dispatch(a)) /\ WF_vars(Die(a)) /\ WF_vars(EarlyFire(a)) /\ WF_vars(Publish(a)) + /\ WF_vars(Discharge(a)) /\ WF_vars(\E n \in inflight : Deliver(n)) /\ \A r \in Reqs : WF_vars(Enqueue(r)) @@ -473,19 +520,32 @@ TypeOK == /\ mstate \in [MonIds -> {"unused", "active", "fired", "cancelled", "late"}] /\ \A r \in Reqs : req[r].s \in {"idle", "sending", "queued", "exiting"} \cup Done /\ \A a \in Actors : (st[a] = "run") <=> (why[a] = None) + /\ disch \in [Actors -> BOOLEAN] /\ sigAfterD \in BOOLEAN + /\ \A a \in Actors : disch[a] => Past(a, st[a]) -\* A queued request is in exactly one live mailbox; past D it was answered -\* (PR #182; rank 11 death boundary). +\* A queued request is in exactly one mailbox, of an actor not yet past its +\* discharge: nothing is admitted after D, and every request queued before +\* it is answered before publication (PR #182; rank 11 death boundary). +Undischarged(a) == st[a] \in {"run", "stopping"} \/ (st[a] = "gone" /\ ~disch[a]) NoOrphanRequest == \A r \in Reqs : req[r].s = "queued" => - /\ st[req[r].t] \in {"run", "stopping"} + /\ Undischarged(req[r].t) /\ Len(SelectSeq(mbox[req[r].t], LAMBDA x : x.k = "req" /\ x.q = r)) = 1 +\* Publication (join, wait_exit, the notices taken at D) comes after every +\* queued and exit-pending request was answered. +AnsweredBeforePublish == + \A r \in Reqs : req[r].s \in {"queued", "exiting"} => st[req[r].t] # "dead" + +\* An actor's exit signals all take effect before its D (the signal gate). +SignalsBeforeDeath == ~sigAfterD + \* An exit error reaches a caller only at D, with the final reason; until \* then a request whose handler panicked is held (rank 11 slot) open. StoppedAtDeath == \A r \in Reqs : - /\ req[r].s = "exiting" => st[req[r].t] = "stopping" + /\ req[r].s = "exiting" => + (st[req[r].t] = "stopping" \/ (st[req[r].t] = "gone" /\ ~disch[req[r].t])) /\ req[r].s = "stopped" => Past(req[r].t, st[req[r].t]) /\ req[r].why = why[req[r].t] \* No message is dispatched once the actor observed its stop (PR #185). @@ -553,7 +613,8 @@ Safety == /\ TypeOK /\ NoOrphanRequest /\ NoDispatchAfterStop /\ DownAtMostOnce /\ NoProcIffLate /\ TrapRespected /\ NormalSignalIgnored /\ KilledFromKill /\ NoEdgeToDead /\ ExitAfterMessages /\ CancelFenced /\ StoppedAtDeath - /\ RuntimeDropReason /\ SkipHook /\ LateSignalWins + /\ RuntimeDropReason /\ SkipHook /\ LateSignalWins /\ AnsweredBeforePublish + /\ SignalsBeforeDeath ---------------------------------------------------------------------------- (* Liveness (under FairSpec: handlers, stopped() and destruction return) *) diff --git a/verification/tla/MCCoordinator.cfg b/verification/tla/MCCoordinator.cfg index e3a7bfdb5..5097b0255 100644 --- a/verification/tla/MCCoordinator.cfg +++ b/verification/tla/MCCoordinator.cfg @@ -2,6 +2,8 @@ SPECIFICATION FairSpec INVARIANT Safety INVARIANT OrderedShutdown INVARIANT JoinAfterAllSignalled +INVARIANT KillOnlyAtGrace +INVARIANT GraceKillsRemaining PROPERTY CleanupCompletes CONSTANTS TargetCount = 3 diff --git a/verification/tla/MCDeferredJobsHeld.cfg b/verification/tla/MCDeferredJobsHeld.cfg new file mode 100644 index 000000000..a189b64b3 --- /dev/null +++ b/verification/tla/MCDeferredJobsHeld.cfg @@ -0,0 +1,14 @@ +\* Fast instance with delayed emits and held results (CI, every PR): three +\* unkeyed jobs, two permits, one instance, one emit per job, one credit. +\* Checks that a held result keeps its credit and its armed deadline. +SPECIFICATION FairSpec +INVARIANT Safety +PROPERTY JobReported +CONSTANTS + J = 3 + Keyed = {} + JobBound = 2 + MaxGen = 1 + MaxEmit = 1 + Credits = 1 + Fault = "None" diff --git a/verification/tla/MCDeferredJobsHeld.tla b/verification/tla/MCDeferredJobsHeld.tla new file mode 100644 index 000000000..911a435c5 --- /dev/null +++ b/verification/tla/MCDeferredJobsHeld.tla @@ -0,0 +1,3 @@ +------------------------ MODULE MCDeferredJobsHeld ------------------------ +EXTENDS DeferredJobs +============================================================================= diff --git a/verification/tla/MCLifecycle.cfg b/verification/tla/MCLifecycle.cfg index 3fcfcd6fa..04974cea4 100644 --- a/verification/tla/MCLifecycle.cfg +++ b/verification/tla/MCLifecycle.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleDeep.cfg b/verification/tla/MCLifecycleDeep.cfg index 763ae745a..c8eacadf3 100644 --- a/verification/tla/MCLifecycleDeep.cfg +++ b/verification/tla/MCLifecycleDeep.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleFence.cfg b/verification/tla/MCLifecycleFence.cfg index 988fb4aed..edd014fe4 100644 --- a/verification/tla/MCLifecycleFence.cfg +++ b/verification/tla/MCLifecycleFence.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleLive.cfg b/verification/tla/MCLifecycleLive.cfg index bccbcf9eb..048783439 100644 --- a/verification/tla/MCLifecycleLive.cfg +++ b/verification/tla/MCLifecycleLive.cfg @@ -35,3 +35,5 @@ PROPERTY DownDelivered PROPERTY LinkedPeersStop INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleLiveDeep.cfg b/verification/tla/MCLifecycleLiveDeep.cfg index c4cfc3d0a..769b62134 100644 --- a/verification/tla/MCLifecycleLiveDeep.cfg +++ b/verification/tla/MCLifecycleLiveDeep.cfg @@ -35,3 +35,5 @@ PROPERTY DownDelivered PROPERTY LinkedPeersStop INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/MCLifecycleWideDeep.cfg b/verification/tla/MCLifecycleWideDeep.cfg index 59ec56ed0..df72af047 100644 --- a/verification/tla/MCLifecycleWideDeep.cfg +++ b/verification/tla/MCLifecycleWideDeep.cfg @@ -30,3 +30,5 @@ INVARIANT StoppedAtDeath INVARIANT RuntimeDropReason INVARIANT SkipHook INVARIANT LateSignalWins +INVARIANT AnsweredBeforePublish +INVARIANT SignalsBeforeDeath diff --git a/verification/tla/README.md b/verification/tla/README.md index e7e982236..ed87475da 100644 --- a/verification/tla/README.md +++ b/verification/tla/README.md @@ -15,9 +15,9 @@ this fits with Loom, Shuttle, Miri, ThreadSanitizer and mutation testing. | `Introspection.tla`, `MCIntrospection.tla/.cfg` | Introspection v1 design model, revision 4 (not implemented, [design](../../docs/design/introspection.md)): one actor whose every activity (`started()`, messages, the status turn, `stopped()`) opens a turn id on an activity word; activities that may suspend forever; panics in `started()`, handlers and `stopped()`, and a caught status-callback panic; the status request with a deadline, callback admission, answer and turn end as separate steps, and the skip of abandoned callbacks; stop, unfenced kill, the runtime drop's swap and D as separate steps, publication; the watchdog's observe, CAS (successful and failed) and record. Each action is one implementation linearization point; ghosts are derived from state changes, never from the fault selector. Twenty named negative controls. 47,968 states (4 messages: 80,482). | | `Coordinator.tla`, `MCCoordinator.tla/.cfg` | System shutdown coordinator: ordered stop of admitted roots, panic fallback that signals all remaining roots, records failure and retains runtime ownership until joins; ordered-shutdown invariants, cooperative cleanup liveness and four negative mutations. | | `NestedSupervision.tla`, `MCNested.tla/.cfg` | Dynamic subtree ownership and shutdown delivery; parent scope completion waits for descendant completion. Bounded three-node/two-generation model with fair cleanup. | -| `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with unwind-local drops, a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Thirteen named negative controls. | -| `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits. Nine named negative controls. | -| `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Seventeen negative-control faults (17 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point and a parent exit reaching `exit_received`. Per PR: `MCLifecycle` (7.5M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | +| `DeferredReplies.tla`, `MCDeferredReplies*.tla/.cfg` | Rank 11 design model (not implemented), part 1: reply slots, send-time deadlines with shortening, deferral credits shared by a token and its first linked job, the exiting phase with exit-pending drops (unwound or dropped while exiting, answered at death with the final reason), a stuck phase (cleanup deadline passed while `stopped()` runs: nothing closed early, death later or never), death, and downstream request cleanup. Fourteen named negative controls. | +| `DeferredJobs.tla`, `MCDeferredJobs*.tla/.cfg` | Rank 11 design model, part 2: pipe jobs with separate execution, cancellation and notification states; keyed replacement including notified jobs; optional job deadlines; a FIFO mailbox with emit credits and the terminal message last; `ActorId` fencing. `MCDeferredJobsEmits` is the fast configuration with emits; `MCDeferredJobsHeld` the one with delayed emits (`emit_at`) and held results (completion, held notification, scheduled emits and credit ownership as separate steps). Eleven named negative controls. | +| `Lifecycle.tla`, `MCLifecycle{,Live,Fence,Deep,LiveDeep,WideDeep}.tla/.cfg` | Base actor lifecycle as proposed for v0.8 ([design](../../docs/design/otp-exit-signals.md)): links, monitors, `trap_exit`, exit signals (`Killed`, ignored `Normal`, `NoProc`, parent exits), administrative stops, requests around death, `Exit`/`Down` ordering. Nineteen negative-control faults (19 safety checks, 2 liveness checks): the v0.7 exit-signal behaviours (direct stops ignoring `trap_exit`, kills ending `Kill`, unfenced `unlink`/`demonitor`, real reasons for late registrations, `stopped()` after kills and untrapped signals, late signals ignored during exit), the PR #182/#185 bugs, early exit errors, kill and panic precedence, runtime-drop reasons, early notification, a split death point, a parent exit reaching `exit_received`, an exit signal past the sender's death point (`NoSignalFence`) and publication before the queued requests were answered (`PublishBeforeDischarge`). Death is two steps, as in `actor_cleanup.rs`: the death point (`Die`/`RuntimeDrop`: signal fence, seal, table closed) and `Discharge` (queued and exit-pending requests answered), with other actors' steps in between; `Publish` waits for `Discharge`. Per PR: `MCLifecycle` (10.0M states, fewer reasons, where the safety controls run), `MCLifecycleFence`, `MCLifecycleLive`; weekly: `MCLifecycleWideDeep` (the former 43M-state per-PR instance), `MCLifecycleDeep`, `MCLifecycleLiveDeep`. | | `Start.tla`, `MCStart{,Names,Cap,Timeout,Continue,Scope,Live,Deep}.tla/.cfg` | Unified start API, 0.9 design (approved after three review rounds) ([design](../../docs/design/unified-start.md), not implemented; TLC-only in `model-map.toml` until trace validation lands with the implementation): `spawn`, `start`, `start_link`, `start_monitor`; placement inheritance; an atomic stage-1 commit with the owner gate; an immutable startup commitment and a waiter; death as D, release, then publication, for the child and the parent; the final reason fixed at D; fallible init with held-signal precedence; the continuation; S8 scope ownership with the polite Shutdown and the scope deadline. 40 negative controls (37 safety, 3 liveness); see [Unified start](#unified-start-09-design). Per PR: `MCStart`, `MCStartNames`, `MCStartCap`, `MCStartTimeout`, `MCStartContinue`, `MCStartScope`, `MCStartLive` (CI leg `start`); weekly: `MCStartDeep`. | | `TraceCohorts.tla`, `TraceNested.tla` | Policy and ownership projections of real cohort/nested Rust executions. | | `trace_check.py`, `negative_checks.py` | Replay recorded traces; deliberately wrong selections, premature resume/join and broken propagation must be rejected or produce counterexamples. | @@ -53,7 +53,7 @@ runs real actors through the scenarios below and writes each trace as JSON; TLC then checks that each trace is a behavior of `Supervision.tla`. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace nix run .#check-tla -- --traces-only target/lifecycle-traces --negative-controls # or --traces DIR to also model-check nix develop -c python3 verification/tla/trace_check.py target/lifecycle-traces ``` @@ -66,8 +66,11 @@ keeps the generated modules and TLC logs. Scenarios: normal start and stop of three children with mixed modes; Permanent and Transient panics and restarts; Transient normal exit (no restart); Temporary exit; `ChildHandle::shutdown` of running children; -factory error then success; non-restartable startup failure; restart budget -exhaustion and disabled retries; shutdown during backoff; startup timeout; +replacement factory error then success; first-start factory error and +first-start timeout (both fail the start, S1); a ready child's exit deferred +during the initial start (`DeferExit`/`Undefer`, S1); non-restartable startup +failure; restart budget exhaustion and disabled retries; shutdown during +backoff; replacement startup timeout; `Supervisor::shutdown` with a short deadline and a hanging `stopped()` (`ShutdownDeadline`, `SupChildTimeout`, `CleanupStuck`); and seeded random crashes, normal exits and external shutdowns racing a stop or shutdown. @@ -137,10 +140,21 @@ in the same PR; the `model-sync` CI job enforces it Trace validation says every recorded run is a behavior of the model. Trace coverage asks the converse: which parts of the model's reachable behavior do the recorded runs never exercise? `trace_coverage.py` reports it per model with -trace validation (`Supervision`, `Cohorts`, `NestedSupervision`). +trace validation (`Supervision`, `Cohorts`, `NestedSupervision`, and by +action keys only `Lifecycle`, `DeferredJobs`, `DeferredReplies`). + +**Action-only models.** For `Lifecycle.tla`, `DeferredJobs.tla` and +`DeferredReplies.tla` the keys are the log-event arms of their trace spec +(`TraceLifecycle.tla`, `TraceJobs.tla`, `TraceReplies.tla`; each arm is one +model action or a named group of them, `A|Send`, `A|Fire`, ...): the model +side is the set of arms, a trace covers the arms its accepted replay takes, +and no state space is explored for them. `coverage-baseline.json` and the +`coverage_floor` of `verification/model-map.toml` hold them like the others. +Model actions without an arm are listed in the map's `[trace.unlogged]` with +the reason. ```bash -cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace +cargo test -p spawned-concurrency --features lifecycle-trace,failpoints --test supervision_trace --test cohort_trace --test tree_trace --test nested_supervision --test coverage_trace --test job_trace --test reply_trace --test lifecycle_trace nix develop -c python3 verification/tla/trace_coverage.py --traces dedicated=target/lifecycle-traces \ --model-keys verification/tla/coverage-model.json.gz --baseline verification/tla/coverage-baseline.json # every test binary: each finished supervisor dumps its traces @@ -411,7 +425,7 @@ and `--negative-controls` removes one guard at a time and requires induction to fail: for `Supervision` the retry limit, ordered start, join wait, restart table, cleanup, unpublish-on-stop, admission gate on `Attempt` and the reverse-shutdown wait; for `Cohorts` and `NestedSupervision` the existing -`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `Join`); +`Fault` mutations (`Selection`, `Cleanup`, `Retirement`, `Closed`, `InPlaceRetry`, `FirstStartRetry`, `PerChildCharge`, `Join`); for `Coordinator` see below. The type annotations are `\* @type:` comments, ignored by TLC. Apalache also @@ -439,8 +453,11 @@ existing reason model. A negative control erases Kill and must reject the resulting transient restart; the entry point adds no new model transition. `queued_drop_panic` checks a queued destructor fault during ordinary stop and the resulting transient restart. Erasing its Panic exit must reject the tape. -`queued_drop_preserves_shutdown` retains a committed Shutdown despite a destructor -failure; escalating it to Panic must reject transient retirement. +`queued_drop_shutdown_restarts` keeps the raw Shutdown reason but records the +destructor failure after the death point as a cleanup failure, which the trace +projects to the abnormal class (v0.8, otp-exit-signals.md "Phases of an +exit"): the Transient child restarts. Projecting the raw Shutdown instead must +reject the restart. All library crates continue to forbid unsafe Rust; native synchronization and Linux affinity use safe interfaces. No unsafe exemption was introduced. @@ -492,9 +509,41 @@ message is dispatched after stop; each monitor's `Down` is delivered at most once and, unless its observer died, exactly once; linked non-trapping peers of an abnormal exit stop; and the exit-signal rules of rank 5e. Reintroducing the PR #182 bug (`Fault = "Retain182"`) and the PR #185 bug (`"Dispatch185"`) each -produce a counterexample (`negative_checks.py`). Still to do with the -implementation: lifecycle-trace events for exit signals, links, monitors and -requests so real runs replay against it, as supervision traces do. +produce a counterexample (`negative_checks.py`). + +**Lifecycle trace validation.** With `lifecycle-trace`, the runtime records +one `LifeEvent` per `Lifecycle.tla` action for the actors spawned while +`lifecycle_trace::record_lifecycle()` is on (both `tasks` and `threads`): +`Trap`, `Send` (a request from `Ext`, admitted or `NoProc`), `Dispatch` (with +the mailbox item kind), `Panic`, `Stop`, `Signal` (also a supervisor's +Shutdown or kill), `Link`/`Unlink`/`Monitor`/`Demonitor`, `Deliver` (a link +notice or `Down` applied by its receiver), `Hook` (`stopped()` runs or is +skipped), `Die` (D, final reason), `Drop` (runtime drop) and `Publish`. Each +is recorded in the critical section of the change it describes: the +dispatch with the run loop's stop check, the stop with its cancel, the +admission check with the push, D with the exit-table close. A +recorded actor's sections take the process-wide trace section first, so the +log is a linearization. `TraceLifecycle.tla` replays it with no unlogged +steps; its header lists the mapping (actors `a1`, `a2` in spawn order, +`start_linked` parents as `LinkParent`, every send a request from `Ext`, +folded request panics, the late `NoProc` notice of a non-trapping caller, +and `Hook` "skip" accepted after a kill that arrived before `stopped()` +began, which the model's `Kill` leaves out). Phase 0 (signals held during +`started()`; not in `Lifecycle.tla` either: actors start in `run`), managed +scopes' own stops and actor-to-actor sends are not mapped: they reject. `concurrency/tests/lifecycle_trace.rs` records 14 +scenarios (`lifecycle_*.json`, schema `lifecycle-v1`): kill with a queued +request answered `Killed` and a later `NoProc`, kill during `stopped()`, +untrapped and `Normal` signals, trapped exits (also from an actor, then a +kill), late link/try_link/monitor (`NoProc`), a late link ending a +non-trapping caller, the unlink and demonitor fences between D and +publication, a monitor's `Down`, a `start_linked` parent exit to a trapping +child, a link notice ending a non-trapping peer, runtime drop (`Killed`), and +one threads-mode run. Controls (`lifecycle_trace_controls`): `stopped()` +after a kill or after an untrapped signal, `Kill` instead of `Killed`, +publication before D, a `Normal` signal stopping an actor, a late `Down` or +link notice with the real reason, a notice after unlink, a `Down` after +demonitor, a parent exit dispatched as an `Exit`, and a runtime drop ending +`Normal`: all eleven are rejected. Actor-to-actor messages (`Send`, recorded with their sender) add three controls: a send after the sender's death point, the death notice overtaking the earlier message, a message dispatched as an outside request. `trace_coverage.py` covers this schema by action keys (see [Trace coverage](#trace-coverage)). Source: [verification strategy](https://github.com/lambdaclass/spawned/blob/efba24340a01df82f872ec422b9dbaf48f897b04/legacy/docs/research/verification-strategy.md#acceptance). `Coordinator.tla` checks coordinator failure ownership independently of the actor @@ -530,8 +579,11 @@ turns, a running-linked-job count per request, and downstream actor B. open and no other job of that request runs. - **Job notifications** hand the token back to a turn; after a job deadline the work keeps running. -- **Exit.** While exiting, stored tokens can still be answered and drops - answer `Stopped`. At the cleanup deadline, a still-running `stopped()` +- **Exit.** While exiting, stored tokens can still be answered. A token + dropped while exiting, or unwound by a panicking turn, is *exit pending* + (`pendA`): it stays open and death answers it `Stopped` with the final + reason (rank 5e, [otp-exit-signals.md](../../docs/design/otp-exit-signals.md#required-changes-to-rank-11-221-design-239-implementation)), + unless the caller's deadline or abandonment comes first. At the cleanup deadline, a still-running `stopped()` makes A stuck: nothing is closed early, `stopped()` may still reply, and callers are bounded by their own deadlines. Only death closes the registry and discards the mailbox. @@ -543,7 +595,10 @@ Invariants: - `AtMostOneDischarge`, `NoOrphan`, `NoSuccessAfterDeadline`, `TimelyExpiry`, `DeadlineWithinCaller`; - `BoundedA` (combined credits) and `BoundedB`; -- `PanicStopped`, `NoEarlyClose`, `DownstreamOwned`, `CallerConsistent`. +- `PanicStopped`, `StoppedAtDeath` (a `Stopped` discharge happens only in + the death step, where the reason is final; the model has no reasons, so + this is how it shows callers get the final one), `NoEarlyClose`, + `DownstreamOwned`, `CallerConsistent`. Liveness is `EveryAcceptedDischarged` and `CallerNeverHangs`, with fairness only on the clock and the deadline service. @@ -571,6 +626,10 @@ exactly the property it targets: | Model (configuration) | Mutation | Breaks | |---|---|---| +| `Supervision` (`MCSupervision`, a rewrite of `Supervision.tla`: no `Fault` constant) | `FirstStartRetry` (v0.7 retried a failed first start) | `RestartFollowsPolicy` | +| `Supervision` (`MCSupervision`, rewrite) | `NoDeferral` (v0.7 decided on a ready child's exit during the initial start) | `RestartFollowsPolicy` | +| `Cohorts` (`MCCohortsAll`) | `FirstStartRetry` | `FirstStartOnce` (in `Safety`) | +| `Cohorts` (`MCCohortsAll`) | `PerChildCharge` (v0.7 charged the restart budget per rebuilt child, not per round) | `RoundCharged` | | `DeferredReplies` | `NoAutoReply` | `NoOrphan` | | `DeferredReplies` | `NoAtomicExpiry` (abstract double discharge; the real race needs Loom) | `AtMostOneDischarge` | | `DeferredReplies` | `LateSuccess` | `NoSuccessAfterDeadline` | @@ -584,6 +643,9 @@ exactly the property it targets: | `DeferredReplies` | `AttachClosed` | `BoundedA` | | `DeferredReplies` | `LeakDownstream` | `DownstreamOwned` | | `DeferredReplies` | `EarlyClose` (closes the registry at the cleanup deadline) | `NoEarlyClose` | +| `DeferredReplies` | `EarlyStopped` (rank 11 before 5e: a token dropped while exiting or unwound answers `Stopped` at once) | `StoppedAtDeath` | +| `DeferredReplies` | `NoReplyFence` (a stored token's reply is admitted after the death point) | `NoReplyAfterDeath` | +| `DeferredReplies` | `CallbackBeforeDischarge` (an answer's wake-up runs before the last answer of the death batch) | `WokenAfterDischarge` | | `DeferredJobs` | `NoInstanceCheck` | `NoLateResult` | | `DeferredJobs` | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` | `DoubleTerminal` | `AtMostOneTerminal` | @@ -593,18 +655,82 @@ exactly the property it targets: | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoCancelFence` | `NoStaleAfterCancel` | | `DeferredJobs` (`MCDeferredJobsEmits`) | `EmitAfterTerminal` | `TerminalLast` | | `DeferredJobs` (`MCDeferredJobsEmits`) | `NoEmitCredit` | `EmitCreditsBounded` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `EarlyCredit` (a held result returns its credit when the work ends) | `HeldCounted` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `HeldDeadlineLost` (the deadline cannot take a held result) | `DeadlineArmedWhileHeld` | +| `DeferredJobs` (`MCDeferredJobsHeld`) | `ExitKeepsHeld` (owner exit cancels running work only, not held results or scheduled emits) | `ExitCancelsJobs` | +| `DeferredJobs` | `NoAdmissionFence` (a job admitted after the exit cancelled the job set) | `ExitCancelsJobs` | -Measured 2026-11-10 (TLC 2.19, fede): +Measured 2026-11-10 (TLC 2.19, fede; the two `MCDeferredReplies` rows on +2026-11-11, after the exit-pending change): | Configuration | Distinct states | Generated | Depth | Time | |---|---|---|---|---| -| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness) | 2,932,280 | 16,385,809 | 33 | 56 s, 10 workers | -| `MCDeferredRepliesDeep` (clock 3; safety) | 8,172,070 | 45,538,582 | 34 | 21 s, 10 workers | +| `MCDeferredReplies` (fast: 2 requests, `BoundA` 2, `BoundB` 1, clock 2; safety and liveness; death split into D and the answer batch, 2026-11-11) | 4,864,810 | 25,029,343 | 34 | 4 min 22 s, 4 workers (fede shared) | +| `MCDeferredRepliesDeep` (clock 3; safety) | 11,016,668 | 58,096,711 | 34 | 55 s, 4 workers | | `MCDeferredJobs` (fast: 3 jobs, 2 keyed, 2 permits, 2 instances, no emits; safety and liveness) | 229,070 | 487,283 | 17 | 5 s, 8 workers | -| `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 20,422 | 37,220 | 18 | 1 s, 8 workers | -| `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit each; safety) | 5,876,658 | 12,792,039 | 23 | 25 s, 12 workers | +| `MCDeferredJobsEmits` (fast: 2 keyed jobs, 1 instance, 2 emits each, 1 credit; safety and liveness) | 24,842 | 50,944 | 18 | 1 s, 2 workers | +| `MCDeferredJobsHeld` (fast: 3 unkeyed jobs, 2 permits, 1 instance, 1 emit or scheduled emit each, 1 credit; safety and liveness) | 1,946,047 | 6,230,609 | 22 | 1 min 30 s, 4 workers (with `ExitCancelsJobs`, 2026-11-11) | +| `MCDeferredJobsDeep` (3 jobs, 2 instances, 1 emit or scheduled emit each; safety) | 7,980,942 | 19,676,218 | 23 | 82 s, 2 workers | + +All 24 named controls find their counterexample, each in a few seconds. + +### Job trace validation + +With `lifecycle-trace`, for owners a test opts in with +`lifecycle_trace::watch_deferred(id)`, the job core (`concurrency/src/reply/jobs.rs`) records +one `DeferredJobs.tla` action per owner instance: `Start` (keyed, with a job +deadline), `Emit`, `Finish` (whether it enqueued the terminal message), +`JobDeadline`, `Cancel`, `Exit` and `Dispatch` (kind and fence verdict). Each +is recorded in the trace section together with its effect (the flag change +under the job lock, the enqueue, the fence read), so the log is a +linearization. `concurrency/tests/job_trace.rs` records seven scenarios (plain +result, a finished keyed job replaced and fenced, emits with one credit, +handle cancel, job deadline with the work still running, owner exit with a +running job, a blocking job emitting, and delayed emits: a held result +released by its last fire in both modes, an emit already due, the job +deadline taking a held result, a cancel dropping a scheduled emit); +`trace_check.py` checks them (schema +`jobs-v1`) against `TraceJobs.tla` with one owner instance and one key. The +permit bound is not checked (`JobBound = J`): the code returns a job's credit +just before `Finish`, and rejected admissions are not model actions. -All 22 named controls find their counterexample, each in a few seconds. +```bash +cargo test -p spawned-concurrency --features lifecycle-trace --test job_trace +python3 verification/tla/trace_check.py target/lifecycle-traces +``` + +`negative_checks.py` requires twelve edited job traces to be rejected (six +for delayed emits: a release before the last fire, a terminal while an emit +is scheduled, a release by a fire that did not take the result, a delayed +emit after the deadline took the result, a fire without a schedule, a drop +while the job still accepts), and the first six: the +replaced job's terminal message delivered, an emit after the terminal +message, a second emit with its credit in flight, a second terminal message +after the deadline's, a quiet finish without a cancel, and a dispatch after +the owner's exit. + +### Reply trace validation + +The reply transport records `DeferredReplies.tla` actions per owner +instance: `Send` (slot created), `Dispatch` (its turn starts), `Discharge` +(the slot's one `OPEN` -> terminal transition, recorded in the trace section +together with the CAS in `Slot::cas`; a reply counts at `WRITING`), `Exit` +(after the registry commits the turn's drops) and `Death` (the closes that +death and mailbox disposal perform are that one model step and are not +logged). `TraceReplies.tla` takes the turn-internal steps that leave no trace +silently (a `Defer` that takes a credit, `Store`, `DropInTurn`, a turn end +that answers nothing, a drop while exiting that makes the slot exit pending, +the clock); a `Discharge` must be the model action that +discharges exactly that request with that kind. Linked jobs and the +downstream actor are not traced (their notifications and job ends are not +logged). `concurrency/tests/reply_trace.rs` records nine scenarios (reply in +the turn, stored then answered or dropped from a later non-request turn, an +unused obligation, `Overloaded` at the bound, deadline expiry by the +service, the caller giving up, death with a stored token, a panicking turn); +`negative_checks.py` requires five edited traces to be rejected (`Overloaded` +below the bound, a reply after death, a second discharge, a drop before the +turn, and a `Stopped` discharge of the panicking turn's token before its +`Exit`, the rank 11 behaviour before 5e). ## Unified start (0.9 design) diff --git a/verification/tla/Supervision.tla b/verification/tla/Supervision.tla index 235852aa4..9f5e93f0e 100644 --- a/verification/tla/Supervision.tla +++ b/verification/tla/Supervision.tla @@ -52,17 +52,18 @@ Restarts(m, r) == ELSE \/ m = "Permanent" \/ m = "Transient" /\ r \in {"none", "Panic", "Kill"} -\* Whether a controller retries after cleanup. A generation that never became -\* ready after the child's initial readiness is a failed restart attempt, -\* retried whatever its exit reason (as OTP retries a failed restart), unless -\* Temporary. Before initial readiness the restart table applies, so a -\* Transient child stopping Normal or Shutdown in its first startup fails it. +\* Whether a controller retries after cleanup. A child's first start that +\* fails (init FALSE) is never retried, whatever its restart type: it fails +\* the supervisor's start (S1 in docs/design/otp-exit-signals.md, OTP's +\* failed_to_start_child). After it, the restart table applies, and a +\* generation that never became ready is a failed restart attempt, retried +\* whatever its exit reason (as OTP retries a failed restart), unless +\* Temporary. Retries(m, r, ready, init) == - \/ Restarts(m, r) - \/ ~ready /\ init /\ m # "Temporary" + init /\ (Restarts(m, r) \/ (~ready /\ m # "Temporary")) Pcs == {"idle", "top", "backoff", "attempt", "constructing", "running", - "cleanup", "decide", "finish", "finished"} + "cleanup", "decide", "deferred", "finish", "finished"} VARIABLES \* @type: Bool; @@ -189,7 +190,11 @@ LoopTop(c) == /\ UNCHANGED <> \* Reserve capacity before waiting. Cancellation/supersession refunds it; -\* only a committed grant enters the rolling time window. +\* only a committed grant enters the rolling time window. One grant per +\* recovery round (rank 5f, supervision.rs charge_round): under OneForOne a +\* round restarts one child, so it is one grant per restart; the cohort +\* strategies' charge (one per round, none per rebuilt child) is +\* Cohorts.tla's RoundCharged. RetryReserve(c) == /\ pc[c] = "backoff" /\ open /\ c \notin reserved /\ budget < MaxRestarts /\ reserved' = reserved \cup {c} /\ budget' = budget + 1 @@ -292,8 +297,25 @@ CleanupStuck(c) == /\ UNCHANGED <> +\* S1: a ready child's exit during the initial start is recorded; its +\* restart decision waits until the tree commits ready (or the start +\* aborts and the exit is part of the rollback), as OTP handles the queued +\* 'EXIT' after init returns. Cleanup already ran. +DeferExit(c) == + /\ pc[c] = "decide" + /\ open /\ ~sready /\ readyGen[c] + /\ Goto(c, "deferred") + /\ UNCHANGED <> + +Undefer(c) == + /\ pc[c] = "deferred" + /\ sready \/ ~open + /\ Goto(c, "decide") + /\ UNCHANGED <> + Decide(c) == /\ pc[c] = "decide" + /\ ~(open /\ ~sready /\ readyGen[c]) /\ IF ~open THEN Goto(c, "finish") /\ UNCHANGED <> ELSE IF Retries(Mode[c], exitR[c], readyGen[c], initial[c]) THEN Goto(c, "top") /\ UNCHANGED <> ELSE IF ~readyGen[c] THEN Fail /\ Goto(c, "finish") \* non-restartable startup failure @@ -381,7 +403,7 @@ ChildStep(c) == \/ \E r \in StartupResults : StartupFailed(c, r) \/ \E r \in SelfExits : SelfExit(c, r) \/ StopObserved(c) \/ CleanupDone(c) \/ CleanupStuck(c) - \/ Decide(c) \/ Finish(c) + \/ DeferExit(c) \/ Undefer(c) \/ Decide(c) \/ Finish(c) SupStep == \/ SupLaunch \/ SupStartDone \/ SupCancelSeen \/ SupStopTurn @@ -399,6 +421,7 @@ Fairness == /\ WF_vars(RetryReserve(c)) /\ WF_vars(RetryRelease(c)) /\ WF_vars(LoopTop(c)) /\ WF_vars(BackoffDone(c)) /\ WF_vars(Attempt(c)) /\ WF_vars(StopObserved(c)) /\ WF_vars(CleanupDone(c)) + /\ WF_vars(DeferExit(c)) /\ WF_vars(Undefer(c)) /\ WF_vars(Decide(c)) /\ WF_vars(Finish(c)) /\ WF_vars(\E r \in {"Shutdown"} : StartupFailed(c, r)) /\ WF_vars(SupLaunch) /\ WF_vars(SupStartDone) /\ WF_vars(SupCancelSeen) @@ -441,24 +464,29 @@ Safety == NoStartAfterClose == [][\A c \in Children : (gen'[c] > gen[c] \/ (pub'[c] # 0 /\ pub[c] = 0)) => open]_vars -\* The design's restart table (docs/design/execution-groups-supervision.md), -\* stated independently of the Restarts operator the actions use: -\* Permanent restarts after any exit, Transient only after an abnormal exit -\* or failed startup, Temporary never. +\* The design's restart table (docs/design/execution-groups-supervision.md, +\* with S1 of otp-exit-signals.md), stated independently of the Restarts +\* operator the actions use: Permanent restarts after any exit, Transient +\* only after an abnormal exit, Temporary never. A failed startup ("none", +\* "timeout") is restarted only as a failed restart attempt below; a first +\* start is never retried. DesignAllowsRestart(m, r) == CASE m = "Temporary" -> FALSE [] m = "Permanent" -> TRUE - [] m = "Transient" -> r \in {"none", "timeout", "Panic", "Kill"} - -\* A new generation after the first only follows an exit for which the -\* design table allows a restart, or a failed restart attempt (a generation -\* that never became ready after the child's initial readiness) of a child -\* that is not Temporary. + [] m = "Transient" -> r \in {"Panic", "Kill"} + +\* A new generation after the first happens only after the tree committed +\* ready (S1: with ReadyMeansAllStarted, no first start is retried and no +\* exit during the initial start is processed before it completes), and +\* only follows an exit for which the design table allows a restart, or a +\* failed restart attempt (a generation that never became ready after the +\* child's initial readiness) of a child that is not Temporary. RestartFollowsPolicy == [][\A c \in Children : (gen'[c] > gen[c] /\ gen[c] > 0) => - \/ DesignAllowsRestart(Mode[c], exitR[c]) - \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary"]_vars + /\ sready + /\ \/ DesignAllowsRestart(Mode[c], exitR[c]) + \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary"]_vars \* Ready children are stopped in reverse declaration order: a stop turn is \* only given once every later child finished or was abandoned, unless the diff --git a/verification/tla/SupervisionCoverage.tla b/verification/tla/SupervisionCoverage.tla index 81931226a..26c62a4bd 100644 --- a/verification/tla/SupervisionCoverage.tla +++ b/verification/tla/SupervisionCoverage.tla @@ -26,7 +26,8 @@ EXTENDS Supervision, TLC ChildActions == {"LoopTop", "RetryReserve", "RetryRelease", "BackoffDone", "Attempt", "BecomeReady", "StartupFailed", "SelfExit", - "StopObserved", "CleanupDone", "CleanupStuck", "Decide", "Finish"} + "StopObserved", "CleanupDone", "CleanupStuck", "DeferExit", "Undefer", + "Decide", "Finish"} Bucket(b) == IF b = 0 THEN "0" ELSE IF b >= MaxRestarts THEN "max" ELSE "mid" diff --git a/verification/tla/SupervisionInd.tla b/verification/tla/SupervisionInd.tla index 7893dc71e..0903aec7f 100644 --- a/verification/tla/SupervisionInd.tla +++ b/verification/tla/SupervisionInd.tla @@ -84,7 +84,29 @@ H_initial == H_retry == \A c \in Children : /\ pc[c] \in {"top", "backoff", "attempt"} /\ gen[c] > 0 => Retries(Mode[c], exitR[c], readyGen[c], initial[c]) - /\ pc[c] = "decide" => gen[c] > 0 + /\ pc[c] \in {"decide", "deferred"} => gen[c] > 0 + +\* S1: nothing restarts before the tree commits ready. Decide sends a +\* controller back to "top" only with initial[c] (Retries); before sready a +\* child has at most its first generation, whose readiness is initial[c], +\* so Decide's deferral guard (open, ~sready, readyGen) leaves no restart. +\* A deferred controller holds a ready generation's exit. +H_first == \A c \in Children : + /\ initial[c] => gen[c] > 0 + /\ gen[c] <= 1 /\ initial[c] => readyGen[c] + /\ gen[c] > 1 => sready + /\ pc[c] \in {"top", "backoff", "attempt"} /\ gen[c] > 0 => sready + /\ pc[c] = "deferred" => readyGen[c] + +\* A generation under construction is not ready yet (Attempt cleared +\* readyGen). A ready generation leaves "running" only through SelfExit or StopObserved, +\* which record a self-exit reason; "none" and "timeout" are startup failures +\* of a generation that never became ready (S1 removed them from the +\* Transient restart table, so RestartFollowsPolicy needs this). +H_exit == \A c \in Children : + /\ pc[c] = "constructing" => ~readyGen[c] \* Attempt cleared it + /\ pc[c] \in {"cleanup", "decide", "deferred", "top", "backoff", "attempt"} + /\ gen[c] > 0 /\ readyGen[c] => exitR[c] \in SelfExits \* Reverse-order shutdown: while stopping, every child after sidx has \* finished or was abandoned on timeout (cdone and timedOut only grow). @@ -115,7 +137,7 @@ SafetyInd == Helpers == /\ H_gen /\ H_launch /\ H_initial - /\ H_retry /\ H_shutdown /\ H_done /\ H_budget + /\ H_retry /\ H_first /\ H_exit /\ H_shutdown /\ H_done /\ H_budget IndInv == TypeInd /\ H_alive /\ H_pub /\ Helpers /\ SafetyInd @@ -128,8 +150,9 @@ NoStartAfterCloseStep == RestartFollowsPolicyStep == \A c \in Children : (gen'[c] > gen[c] /\ gen[c] > 0) => - \/ DesignAllowsRestart(Mode[c], exitR[c]) - \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary" + /\ sready + /\ \/ DesignAllowsRestart(Mode[c], exitR[c]) + \/ ~readyGen[c] /\ initial[c] /\ Mode[c] # "Temporary" ReverseShutdownStep == \A c \in Children : (stopTurn'[c] /\ ~stopTurn[c] /\ ~deadline') => diff --git a/verification/tla/TraceJobs.tla b/verification/tla/TraceJobs.tla new file mode 100644 index 000000000..04932c033 --- /dev/null +++ b/verification/tla/TraceJobs.tla @@ -0,0 +1,84 @@ +------------------------------ MODULE TraceJobs ------------------------------ +(***************************************************************************) +(* Trace validation for DeferredJobs.tla. *) +(* *) +(* With the `lifecycle-trace` feature, the job core records one event per *) +(* model action for each owner instance (concurrency/src/reply/jobs.rs): *) +(* [a |-> action, j |-> job (1..J in start order), x |-> parameter, *) +(* keyed, dl, fenced]. Each event is recorded in the trace section *) +(* together with its effect (the flag change, the enqueue, or the fence *) +(* read at dispatch), so the log is a linearization. *) +(* *) +(* Start(j) admission (keyed: replaces the key; dl: has a job *) +(* deadline) *) +(* Emit(j) an emit enqueued *) +(* Finish(j) execution ended; x = "notified" if it enqueued the *) +(* terminal message, "quiet" if the notification was *) +(* already taken or suppressed, "held" if scheduled *) +(* emits keep the result *) +(* Schedule(j) emit_at registered a delayed emit (ScheduleEmit) *) +(* Fire(j) a delayed emit fired (FireScheduled): x = "emit" if *) +(* enqueued, "dropped" if not; rel: the last one took a *) +(* held result. A cancel drops the scheduled emits after *) +(* the model's Cancel already did: those "dropped" fires *) +(* are stutter steps *) +(* Release(j) the held result's terminal, after the job's lock *) +(* (ReleaseHeld): x = "notified" or "quiet" *) +(* JobDeadline(j) the job deadline took the notification *) +(* Cancel(j) handle cancel (or the opt-in caller give-up) *) +(* Exit the owner's exit cancelled the job set *) +(* Dispatch(j) the mailbox head, of kind x, fenced or delivered *) +(* *) +(* One trace is one owner instance (MaxGen = 1) and at most one key. The *) +(* model's permit bound is not checked (JobBound = J): rejected *) +(* admissions are not model actions. The model's Restart (a new owner *) +(* instance reading the same slot-addressed mailbox) has no code *) +(* counterpart: a job's messages go to its owner instance's own mailbox, *) +(* which is discarded at that instance's death, so a trace is one *) +(* instance. The trace is accepted iff some behavior consumes the whole *) +(* log (TLC reports TraceIncomplete violated). *) +(***************************************************************************) +EXTENDS DeferredJobs, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +Step(e) == + CASE e.a = "Start" -> e.j \in Jobs /\ Start(e.j) /\ hasDl'[e.j] = e.dl + /\ (e.j \in Keyed) = e.keyed + [] e.a = "Emit" -> e.j \in Jobs /\ Emit(e.j) + [] e.a = "Finish" -> e.j \in Jobs /\ Finish(e.j) + /\ (termSent'[e.j] > termSent[e.j]) = (e.x = "notified") + /\ held'[e.j] = (e.x = "held") + [] e.a = "Schedule" -> e.j \in Jobs /\ ScheduleEmit(e.j) + [] e.a = "Fire" -> e.j \in Jobs /\ + \/ /\ FireScheduled(e.j) + /\ (inflight'[e.j] > inflight[e.j]) = (e.x = "emit") + /\ releasing'[e.j] = e.rel + \/ /\ e.x = "dropped" /\ ~e.rel + /\ cancelled[e.j] /\ sched[e.j] = 0 + /\ UNCHANGED vars + [] e.a = "Release" -> e.j \in Jobs /\ ReleaseHeld(e.j) + /\ (termSent'[e.j] > termSent[e.j]) = (e.x = "notified") + [] e.a = "JobDeadline" -> e.j \in Jobs /\ JobDeadline(e.j) + [] e.a = "Cancel" -> e.j \in Jobs /\ Cancel(e.j) + [] e.a = "Exit" -> Exit + [] e.a = "Dispatch" -> /\ mbox # <<>> + /\ Head(mbox).j = e.j /\ Head(mbox).kind = e.x + /\ Fenced(Head(mbox)) = e.fenced + /\ Dispatch + [] OTHER -> FALSE + +TraceNext == i < Len(Trace) /\ Step(Trace[i + 1]) /\ i' = i + 1 + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/TraceLifecycle.tla b/verification/tla/TraceLifecycle.tla new file mode 100644 index 000000000..d74894866 --- /dev/null +++ b/verification/tla/TraceLifecycle.tla @@ -0,0 +1,156 @@ +--------------------------- MODULE TraceLifecycle --------------------------- +(***************************************************************************) +(* Trace validation for Lifecycle.tla (rank 5e exit signals, 5g runtime *) +(* drop; design docs/design/otp-exit-signals.md). *) +(* *) +(* With the `lifecycle-trace` feature, spawned-concurrency records, for *) +(* the actors spawned while recording is on, one event per model action *) +(* (concurrency/src/lifecycle_trace.rs, LifeEvent). Each event is recorded *) +(* inside the critical section of the state change it describes, under *) +(* one process-wide trace section, so the recorded order is a *) +(* linearization: *) +(* Trap ctx.trap_exit, when the flag changes (SetTrap) *) +(* Request, a send or request admitted ("queued") or refused ("noproc") *) +(* Enqueue (one recorded Send; trace_check.py splits it in two) *) +(* Send a plain message from a recorded actor's turn or hook to *) +(* another, admitted (Send; its Dispatch has kind "msg") *) +(* Dispatch the run loop takes the mailbox head of kind k, in the same *) +(* step as its stop check (Dispatch); p: the turn of a request *) +(* panicked (the model's panic branch) *) +(* Panic a panic in a turn, started() or stopped() (SelfExit or *) +(* PanicWhileExiting); p: already taken by its Dispatch *) +(* Stop ctx.stop*, ChildHandle::stop* (SelfExit or AdminStop) *) +(* Signal ctx.exit, ChildHandle::exit/kill (ExitSignal), or a *) +(* supervisor's Shutdown or kill (SupShutdown, SupKill) *) +(* Link link/try_link that installed a record or found the peer *) +(* past D (Link, TryLink); Unlink, Monitor, Demonitor *) +(* Deliver a link notice or Down applied by its receiver (Deliver) *) +(* Hook stopped() begins ("run") or is skipped ("skip") *) +(* Die the death point D with the final reason (Die) *) +(* Drop the runtime dropped the actor before D (RuntimeDrop) *) +(* Publish destruction finished; link notices are delivered after it *) +(* and wait_exit returns after them (Publish) *) +(* *) +(* Identities: the recorded actors are a1, a2, ... in spawn order (their *) +(* start_linked parents give LinkParent; the model starts them all, idle, *) +(* and a Spawn event is not replayed). Outside code is Ext; a supervisor's *) +(* signal is Sup and its target is in Children. A plain message that a *) +(* recorded actor sends to another during a turn or hook (the recorder *) +(* tracks the actor whose turn a thread runs) is the model's Send; every *) +(* other send or request (from outside code, an unrecorded actor, to *) +(* itself, a request, or refused) is a request from Ext. *) +(* *) +(* Mapping notes: *) +(* - Handlers are atomic in the model. Dispatch is recorded when the *) +(* message is taken, and a handler's own actions (Stop, Link, Monitor, *) +(* ...) follow it as separate steps of a running actor. A request turn *) +(* that panics is the model's panic branch of Dispatch only when the *) +(* actor did nothing else in between; otherwise its Dispatch is the *) +(* reply branch and the Panic a SelfExit. *) +(* - A parent exit is the model's Dispatch (which begins exit); the stop *) +(* it requests in code is not recorded. *) +(* - A non-trapping caller of link to an actor past D exits NoProc in the *) +(* model's Link step; the code applies the late notice right after, in *) +(* the same section, so that Deliver is a stutter step here. *) +(* - Hook "skip" is also accepted when the model's hook ghost is TRUE but *) +(* a kill arrived before stopped() began: the code then skips stopped() *) +(* (E1), while Lifecycle.tla's Kill during "stopping" leaves hook as it *) +(* was. Hook "run" requires the ghost (SkipHook). *) +(* - A panic after a kill, or a second panic, changes nothing in the *) +(* model's reason classes and is a stutter step. *) +(* - Not in the mapping (an event the model cannot take rejects the *) +(* trace): phase 0 (signals held while started() runs, recorded as *) +(* Held), managed scopes' own stops, actor-to-actor requests, time. *) +(* *) +(* There are no unlogged steps: every model step is a logged event, so *) +(* TLC replays a trace almost deterministically. The trace is accepted iff *) +(* some behavior consumes the whole log (TraceIncomplete violated). *) +(* verification/tla/trace_check.py (schema lifecycle-v1) generates the *) +(* constants and runs TLC. *) +(***************************************************************************) +EXTENDS Lifecycle, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +DispatchAs(a, k, p) == + /\ mbox[a] # <<>> /\ Head(mbox[a]).k = k + /\ Dispatch(a) + /\ CASE k = "req" -> req'[Head(mbox[a]).q].s = IF p THEN "exiting" ELSE "ok" + \* exit_received's own stop, if any, is a later Stop event + [] k = "exit" -> UNCHANGED life + [] OTHER -> TRUE + +PanicAs(a, folded) == + IF folded THEN st[a] = "stopping" /\ why[a] = "Panic" /\ UNCHANGED vars + ELSE IF st[a] = "run" THEN SelfExit(a, "Panic") + ELSE IF st[a] = "stopping" /\ (killed[a] \/ why[a] = "Panic") THEN UNCHANGED vars + ELSE PanicWhileExiting(a) + +SignalAs(e) == + IF e.s = Sup + THEN IF e.x = "Kill" THEN SupKill(e.t) ELSE e.x = "Shutdown" /\ SupShutdown(e.t) + ELSE ExitSignal(e.s, e.t, e.x) + +LinkAs(e) == + CASE e.x = "try" -> TryLink(e.t, e.s) + [] e.x = "linked" -> ~IsGone(e.s) /\ Link(e.t, e.s) + [] e.x = "noproc" -> IsGone(e.s) /\ Link(e.t, e.s) + [] OTHER -> FALSE + +Matches(n, e) == + /\ n.to = e.t /\ n.from = e.s /\ n.k = e.k /\ n.r = e.x /\ n.o = e.o + /\ (e.k = "down" => n.m = e.m) + +DeliverAs(e) == + \/ \E n \in inflight : Matches(n, e) /\ Deliver(n) + \* The late NoProc notice of a non-trapping caller: applied by Link. + \/ /\ e.k = "link" /\ e.o = "late" /\ e.x = "NoProc" + /\ ~\E n \in inflight : Matches(n, e) + /\ ~trap[e.t] /\ st[e.t] = "stopping" /\ cause[e.t] = "noproc" + /\ UNCHANGED vars + +HookAs(a, x) == + /\ st[a] = "stopping" + /\ IF x = "run" THEN hook[a] ELSE ~hook[a] \/ killed[a] + /\ UNCHANGED vars + +\* The model action named by logged event e, constrained by its parameters. +Step(e) == + CASE e.a = "Trap" -> SetTrap(e.t, e.x = "true") + [] e.a = "Request" -> Request(e.q, e.t) + [] e.a = "Enqueue" -> Enqueue(e.q) /\ req'[e.q].s = e.x + [] e.a = "Send" -> Send(e.s, e.t) + [] e.a = "Dispatch" -> DispatchAs(e.t, e.k, e.p) + [] e.a = "Panic" -> PanicAs(e.t, e.p) + [] e.a = "Stop" -> SelfExit(e.t, e.x) \/ AdminStop(e.t, e.x) + [] e.a = "Signal" -> SignalAs(e) + [] e.a = "Link" -> LinkAs(e) + [] e.a = "Unlink" -> Unlink(e.t, e.s) + [] e.a = "Monitor" -> Monitor(e.t, e.s, e.m) /\ mstate'[e.m] = e.x + [] e.a = "Demonitor" -> Demonitor(e.t, e.m) + [] e.a = "Deliver" -> DeliverAs(e) + [] e.a = "Hook" -> HookAs(e.t, e.x) + [] e.a = "Die" -> why[e.t] = e.x /\ Die(e.t) + [] e.a = "Drop" -> RuntimeDrop(e.t) /\ why'[e.t] = e.x + [] e.a = "Publish" -> Publish(e.t) + [] OTHER -> FALSE + +\* Answering the queued requests after D is not logged: it is taken +\* silently between Die (or Drop) and Publish, which waits for it. +TraceNext == + /\ i < Len(Trace) + /\ \/ Step(Trace[i + 1]) /\ i' = i + 1 + \/ \E a \in Actors : Discharge(a) /\ UNCHANGED i + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/TraceReplies.tla b/verification/tla/TraceReplies.tla new file mode 100644 index 000000000..39f1b8b01 --- /dev/null +++ b/verification/tla/TraceReplies.tla @@ -0,0 +1,118 @@ +----------------------------- MODULE TraceReplies ----------------------------- +(***************************************************************************) +(* Trace validation for DeferredReplies.tla: one owner instance A's reply *) +(* slots, its pipe_reply jobs and their downstream requests to an actor B. *) +(* *) +(* With the `lifecycle-trace` feature, the reply transport records per *) +(* owner instance (concurrency/src/lifecycle_trace.rs, ReplyEvent; `r` is *) +(* the request's slot, numbered by trace_check.py at each Send): *) +(* Send slot created (Send) *) +(* Dispatch its request turn starts (DispatchA) *) +(* Discharge the slot's one OPEN -> terminal transition, in the trace *) +(* section with its CAS; a reply's is OPEN -> REPLIED (x: ok) *) +(* and reads no clock, so "timeout" is always an expiry *) +(* (Expire), never a reply (OTP: the caller's timeout) *) +(* Shorten shorten_deadline lowered the deadline, under the slot lock *) +(* (Shorten; TLC picks the new deadline) *) +(* PipeReply pipe_reply* launched a job ("linked") or the slot refused *) +(* it ("refused": closed, or no credit for a second job) *) +(* JobNotify the job's done or deadline message is dispatched to A, not *) +(* fenced (JobNotify; x: done ends the job, deadline keeps it)*) +(* JobEnd the job's execution ended without taking its notification *) +(* (after a deadline notification, or cancelled) (JobEnds) *) +(* JobCancel the job was cancelled while it still held the token: its *) +(* caller gave up (cancel_on_caller_gone) or A's exit began *) +(* (JobCancel) *) +(* CallB the job's work created a request slot at another actor B *) +(* (JobCallB; a thread-local link set while the work runs) *) +(* BDispatch B deferred that request ("stateB") or answered it in its *) +(* turn ("done": a reply, Overloaded at B's bound, a drop) *) +(* BReply B answered the deferred request later (BReply) *) +(* Exit the registry enters exiting (exit begins, before stopped())*) +(* Stuck a supervisor's cleanup deadline passed before A's death *) +(* point (CleanupDeadline); one after it (a slow destructor *) +(* before publication) is not recorded: the registry is sealed*) +(* Death the death point D: the reply gate is fenced, the registry *) +(* sealed (Death). The answers death then gives and their *) +(* wake-ups (Answer) are not logged and are taken silently. *) +(* *) +(* Turn-internal steps leave no trace and are taken silently: Defer that *) +(* takes a credit (or finds the slot closed), Store, DropInTurn, a turn *) +(* end that answers nothing, a drop while exiting (exit pending, answered *) +(* by Death), and the clock. Sent messages (not requests) leave no trace: *) +(* the model's turns are request turns and JobNotify turns, and replies *) +(* from other turns or stopped() are LaterReply. B's caller-side closes *) +(* (its slot abandoned when the job's future is dropped, or expired) are *) +(* not B's steps: the model folds them into the job's end (EndDown). *) +(* *) +(* Not mapped (an event the model cannot take rejects the trace): *) +(* - pipe_reply from a non-request turn (a stored token), or a token *) +(* shortened inside a job: the model attaches and shortens only tokens *) +(* of the current request turn or stored ones; *) +(* - a linked job cancelled through its JobHandle while its caller still *) +(* waits: the model cancels only on caller give-up or owner exit; *) +(* - a second downstream request from one request's jobs. *) +(* *) +(* trace_check.py instantiates N (requests), BoundA (A's MAX_OUTSTANDING), *) +(* BoundB (B's, default 1) and a clock generous enough for the trace; *) +(* deadlines are chosen by TLC. *) +(***************************************************************************) +EXTENDS DeferredReplies, Sequences, TLC + +CONSTANT Trace + +VARIABLE i + +traceVars == <> + +TraceInit == Init /\ i = 0 + +Only(r, k) == slot[r] = "open" /\ slot'[r] = k + /\ \A q \in Req \ {r} : slot'[q] = slot[q] + +Discharging(r) == + \/ ReplyNow(r) \/ LaterReply(r) \/ LaterDrop(r) \/ Expire(r) + \/ CallerGivesUp(r) \/ Defer(r) \/ EndTurn \/ Panic + +Step(e) == + CASE e.a = "Send" -> e.r \in Req /\ Send(e.r) + [] e.a = "Dispatch" -> e.r \in Req /\ DispatchA(e.r) + [] e.a = "Discharge" -> e.r \in Req /\ Discharging(e.r) /\ Only(e.r, e.x) + [] e.a = "Exit" -> \/ ExitBegin + \/ Panic /\ UNCHANGED slot + \/ phase = "exiting" /\ UNCHANGED vars + [] e.a = "Shorten" -> e.r \in Req /\ Shorten(e.r) + [] e.a = "PipeReply" -> e.r \in Req /\ PipeReply(e.r) + /\ holder'[e.r] = IF e.x = "linked" THEN "doneA" ELSE "localA" + [] e.a = "JobNotify" -> e.r \in Req /\ JobNotify(e.r) + /\ jobs'[e.r] = IF e.x = "done" THEN jobs[e.r] - 1 ELSE jobs[e.r] + [] e.a = "JobEnd" -> e.r \in Req /\ JobEnds(e.r) + [] e.a = "JobCancel" -> e.r \in Req /\ JobCancel(e.r) + [] e.a = "CallB" -> e.r \in Req /\ JobCallB(e.r) + [] e.a = "BDispatch" -> e.r \in Req /\ BDispatch(e.r) /\ down'[e.r] = e.x + [] e.a = "BReply" -> e.r \in Req /\ BReply(e.r) + [] e.a = "Stuck" -> CleanupDeadline + [] e.a = "Death" -> Death + [] OTHER -> FALSE + +\* Steps that leave no trace: they answer no request. +Silent == + /\ \/ Tick \/ EndTurn + \/ \E r \in Req : Defer(r) \/ Store(r) \/ DropInTurn(r) \/ LaterDrop(r) + /\ UNCHANGED slot + +\* The steps of death after D, which leave no trace. +DeathSilent == \/ \E r \in Req : Answer(r) + \/ Discharged + +TraceNext == + /\ i < Len(Trace) + /\ \/ Step(Trace[i + 1]) /\ UNCHANGED <> /\ i' = i + 1 + \/ (Silent \/ DeathSilent) /\ UNCHANGED <> /\ UNCHANGED i + +TraceSpec == TraceInit /\ [][TraceNext]_traceVars + +\* Violated exactly when the whole log has been matched: TLC reporting this +\* invariant as violated means the trace is ACCEPTED. +TraceIncomplete == i < Len(Trace) +============================================================================= diff --git a/verification/tla/TraceSupervision.tla b/verification/tla/TraceSupervision.tla index 9760f7873..a8aa5d2db 100644 --- a/verification/tla/TraceSupervision.tla +++ b/verification/tla/TraceSupervision.tla @@ -33,7 +33,7 @@ Bool(x) == x = "true" ChildEvents == {"RetryReserve", "RetryRelease", "LoopTop", "BackoffDone", "Attempt", "BecomeReady", "StartupFailed", "SelfExit", "StopObserved", "CleanupDone", - "CleanupStuck", "Decide", "Finish"} + "CleanupStuck", "DeferExit", "Undefer", "Decide", "Finish"} \* The model action named by logged event e, constrained by its parameters. Step(e) == @@ -70,6 +70,8 @@ Step(e) == [] e.a = "StopObserved" -> StopObserved(c) [] e.a = "CleanupDone" -> CleanupDone(c) [] e.a = "CleanupStuck" -> CleanupStuck(c) + [] e.a = "DeferExit" -> DeferExit(c) + [] e.a = "Undefer" -> Undefer(c) [] e.a = "Decide" -> /\ Decide(c) /\ CASE e.x = "closed" -> ~open diff --git a/verification/tla/apalache.sh b/verification/tla/apalache.sh index 5881e109e..37c54f172 100755 --- a/verification/tla/apalache.sh +++ b/verification/tla/apalache.sh @@ -134,6 +134,14 @@ for model in $models; do # TemporaryStartsOnce: restart regardless of mode and exit reason. mutate "$module" restart-policy Supervision.tla \ 's/ELSE IF Retries\(Mode\[c\], exitR\[c\], readyGen\[c\], initial\[c\]\) THEN/ELSE IF TRUE THEN/' IndInv + # H_first (S1): v0.7's retry of a failed first start (restart before + # the tree committed ready). + mutate "$module" first-start-retry Supervision.tla \ + 's/ init \/\\ \(Restarts\(m, r\) \\\/ \(~ready \/\\ m # "Temporary"\)\)/ Restarts(m, r) \\\/ (~ready \/\\ init \/\\ m # "Temporary")/' IndInv + # H_first (S1): v0.7's immediate restart of a ready child that exits + # during the initial start (no deferral). + mutate "$module" no-deferral Supervision.tla \ + 's/(Decide\(c\) ==\s*\/\\ pc\[c\] = "decide")\s*\/\\ ~\(open \/\\ ~sready \/\\ readyGen\[c\]\)/$1/s' IndInv # AtMostOneGeneration: a self-exit skips cleanup and goes straight to decide. mutate "$module" skip-cleanup Supervision.tla \ 's/(SelfExit\(c, r\) ==.*?)Goto\(c, "cleanup"\)/$1Goto(c, "decide")/s' IndInv @@ -153,7 +161,7 @@ for model in $models; do Cohorts|Nested) module="MC${model}Ind" if [[ "$model" == Cohorts ]]; then - probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry" + probes=NoMidRoundState faults="Selection Cleanup Retirement Closed InPlaceRetry FirstStartRetry PerChildCharge" else probes=NoFinishedSubtree faults="Join Closed" fi @@ -187,7 +195,9 @@ for model in $models; do # without signalling the remaining targets (JoinAfterAllSignalled); # RollbackReady: the #209 bug, a ready root stopped by its startup # rollback outside the coordinator's order (OrderedShutdown). - for fault in ReleaseEarly ForgetFailure ForgetSignal RollbackReady; do + # NoKillAtGrace / KillEarly: v0.8 grace-expiry kill missing, or sent + # before the grace expired (GraceKillsRemaining, KillOnlyAtGrace). + for fault in ReleaseEarly ForgetFailure ForgetSignal RollbackReady NoKillAtGrace KillEarly; do mutate "$module" "$fault" "$module.tla" \ "s/OVERRIDE_Fault == \"None\"/OVERRIDE_Fault == \"$fault\"/" IndInv 4 done diff --git a/verification/tla/coverage-baseline.json b/verification/tla/coverage-baseline.json index 721b81bad..026760799 100644 --- a/verification/tla/coverage-baseline.json +++ b/verification/tla/coverage-baseline.json @@ -91,5 +91,52 @@ "A|TJoined/leaf", "A|TJoined/owner", "A|TPropagate" + ], + "Lifecycle": [ + "A|Deliver", + "A|Demonitor", + "A|Die", + "A|Dispatch", + "A|Drop", + "A|Enqueue", + "A|Hook", + "A|Link", + "A|Monitor", + "A|Panic", + "A|Publish", + "A|Request", + "A|Send", + "A|Signal", + "A|Stop", + "A|Trap", + "A|Unlink" + ], + "DeferredJobs": [ + "A|Cancel", + "A|Dispatch", + "A|Emit", + "A|Exit", + "A|Finish", + "A|Fire", + "A|JobDeadline", + "A|Release", + "A|Schedule", + "A|Start" + ], + "DeferredReplies": [ + "A|BDispatch", + "A|BReply", + "A|CallB", + "A|Death", + "A|Discharge", + "A|Dispatch", + "A|Exit", + "A|JobCancel", + "A|JobEnd", + "A|JobNotify", + "A|PipeReply", + "A|Send", + "A|Shorten", + "A|Stuck" ] } diff --git a/verification/tla/coverage-model.json.gz b/verification/tla/coverage-model.json.gz index c153a652d..0c244de0b 100644 Binary files a/verification/tla/coverage-model.json.gz and b/verification/tla/coverage-model.json.gz differ diff --git a/verification/tla/negative_checks.py b/verification/tla/negative_checks.py index 289f4f95a..51a3b2855 100644 --- a/verification/tla/negative_checks.py +++ b/verification/tla/negative_checks.py @@ -82,7 +82,7 @@ def model_fault(module, fault, tlc, temporal=False, invariant="Safety", drop=()) with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) cfg = (HERE / f"{module}.cfg").read_text().replace('Fault = "None"', f'Fault = "{fault}"') for name in drop: cfg = cfg.replace(f"INVARIANT {name}\n", "") @@ -103,7 +103,7 @@ def named_fault(module, fault, invariant, tlc): with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) lines = [] for line in (HERE / f"{module}.cfg").read_text().splitlines(): if line.startswith("PROPERTY"): @@ -122,6 +122,50 @@ def named_fault(module, fault, invariant, tlc): print(f"COUNTEREXAMPLE model: {module}/{fault} ({invariant})") +def spec_mutation(module, spec, label, pattern, replacement, prop, tlc): + """Run MODULE with one guard of SPEC rewritten (for models without a + Fault constant), checking TypeOK and the single action property PROP, + and require PROP to be violated.""" + import re + with tempfile.TemporaryDirectory(prefix="bad-spec-") as directory: + work = Path(directory) + for source in HERE.glob("*.tla"): + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) + text = (work / spec).read_text() + mutated, count = re.subn(pattern, replacement, text, flags=re.S) + if count != 1: + raise AssertionError(f"{spec}/{label}: mutation did not apply") + (work / spec).write_text(mutated) + lines = [] + # Keep SPECIFICATION, CONSTANTS and CONSTRAINT (which precede the + # checked invariants and properties in the cfg). + for line in (HERE / f"{module}.cfg").read_text().splitlines(): + if line.startswith(("INVARIANT", "PROPERT")): + break + lines.append(line) + lines += ["INVARIANT TypeOK", f"PROPERTY {prop}"] + (work / f"{module}.cfg").write_text("\n".join(lines) + "\n") + result = subprocess.run(shlex.split(tlc) + ["-workers", "1", "-deadlock", "-cleanup", module], + cwd=work, capture_output=True, text=True) + log = result.stdout + result.stderr + if f"property {prop} is violated" not in log: + raise AssertionError(f"{spec}/{label}: {prop} not violated\n{log}") + print(f"COUNTEREXAMPLE model: {spec}/{label} ({prop})") + + +# S1 (otp-exit-signals.md): v0.7's startup behaviours against Supervision.tla, +# which has no Fault constant: (label, pattern, replacement). +SUPERVISION_S1_CONTROLS = [ + # v0.7 retried a failed first start of a Permanent/Transient child. + ("FirstStartRetry", + r' init /\\ \(Restarts\(m, r\) \\/ \(~ready /\\ m # "Temporary"\)\)', + r' Restarts(m, r) \\/ (~ready /\\ init /\\ m # "Temporary")'), + # v0.7 decided at once on a ready child's exit during the initial start. + ("NoDeferral", + r'(Decide\(c\) ==\s*/\\ pc\[c\] = "decide")\s*/\\ ~\(open /\\ ~sready /\\ readyGen\[c\]\)', + r'\1'), +] + def source_fault(module, source, old, new, invariant, tlc): """Run MODULE (safety only: TypeOK and INVARIANT) with OLD replaced by NEW in SOURCE, for models without a Fault constant, and require INVARIANT to @@ -168,7 +212,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): with tempfile.TemporaryDirectory(prefix="bad-model-") as directory: work = Path(directory) for source in HERE.glob("*.tla"): - shutil.copy(source, work) + shutil.copyfile(source, work / source.name) # writable copy (sources may be read-only) lines = [] for line in (HERE / f"{module}.cfg").read_text().splitlines(): if line.startswith(("PROPERTY", "INVARIANT")): @@ -196,6 +240,7 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCLifecycle", "KillNoSupersede", "KilledFromKill"), ("MCLifecycleFence", "NoUnlinkFence", "CancelFenced"), ("MCLifecycleFence", "NoDemonitorFence", "CancelFenced"), + ("MCLifecycleFence", "NoDemonitorFlush", "CancelFenced"), ("MCLifecycle", "EarlyStopped", "StoppedAtDeath"), ("MCLifecycle", "PanicOverKill", "KilledFromKill"), ("MCLifecycle", "DropKeepsNormal", "RuntimeDropReason"), @@ -207,6 +252,8 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCLifecycle", "DropQueued", "NoOrphanRequest"), ("MCLifecycle", "EarlyNotify", "ExitAfterMessages"), ("MCLifecycle", "SplitDeathPoint", "NoEdgeToDead"), + ("MCLifecycle", "PublishBeforeDischarge", "AnsweredBeforePublish"), + ("MCLifecycle", "NoSignalFence", "SignalsBeforeDeath"), ] # Liveness controls run on a smaller instance of MCLifecycleLive. @@ -230,6 +277,9 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredReplies", "AttachClosed", "BoundedA"), ("MCDeferredReplies", "LeakDownstream", "DownstreamOwned"), ("MCDeferredReplies", "EarlyClose", "NoEarlyClose"), + ("MCDeferredReplies", "EarlyStopped", "StoppedAtDeath"), + ("MCDeferredReplies", "NoReplyFence", "NoReplyAfterDeath"), + ("MCDeferredReplies", "CallbackBeforeDischarge", "WokenAfterDischarge"), ("MCDeferredJobs", "NoInstanceCheck", "NoLateResult"), ("MCDeferredJobs", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobs", "DoubleTerminal", "AtMostOneTerminal"), @@ -239,6 +289,10 @@ def named_temporal(module, fault, prop, tlc, constants=None): ("MCDeferredJobsEmits", "NoCancelFence", "NoStaleAfterCancel"), ("MCDeferredJobsEmits", "EmitAfterTerminal", "TerminalLast"), ("MCDeferredJobsEmits", "NoEmitCredit", "EmitCreditsBounded"), + ("MCDeferredJobsHeld", "EarlyCredit", "HeldCounted"), + ("MCDeferredJobsHeld", "HeldDeadlineLost", "DeadlineArmedWhileHeld"), + ("MCDeferredJobsHeld", "ExitKeepsHeld", "ExitCancelsJobs"), + ("MCDeferredJobs", "NoAdmissionFence", "ExitCancelsJobs"), ] # Unified start (docs/design/unified-start.md): each mutation of the start @@ -319,6 +373,289 @@ def start_controls(tlc): ] +def reply_trace_controls(traces, tlc): + """Edits of recorded reply-slot traces that DeferredReplies.tla must + reject.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "replies-v1"} + + def overload_below_bound(events): + next(e for e in events if e["a"] == "Discharge")["x"] = "overloaded" + require_rejected(by_name["replies_in_turn"], overload_below_bound, + "Overloaded with a free credit", tlc) + + def reply_after_death(events): + held = next(e for e in events if e["a"] == "Send") + events.append({"a": "Discharge", "r": held["r"], "x": "ok"}) + require_rejected(by_name["replies_death"], reply_after_death, + "stored token answered after death", tlc) + + def discharge_twice(events): + first = next(i for i, e in enumerate(events) if e["a"] == "Discharge") + events.insert(first + 1, dict(events[first], x="ok")) + require_rejected(by_name["replies_expired"], discharge_twice, + "second discharge of an expired request", tlc) + + def dropped_before_dispatch(events): + dispatch = next(i for i, e in enumerate(events) if e["a"] == "Dispatch") + drop = next(i for i, e in enumerate(events) if e.get("x") == "dropped") + events.insert(dispatch, events.pop(drop)) + require_rejected(by_name["replies_unused"], dropped_before_dispatch, + "unused obligation dropped before its turn", tlc) + + def stopped_before_death(events): + # Rank 11 before 5e: the token unwound by the panicking turn was + # answered Stopped at once, before the death point fixed the reason. + held = next(e for e in events if e["a"] == "Send") + exit_at = next(i for i, e in enumerate(events) if e["a"] == "Exit") + events.insert(exit_at, {"a": "Discharge", "r": held["r"], "x": "stopped"}) + require_rejected(by_name["replies_panic"], stopped_before_death, + "exit error answered Stopped before the death point", tlc) + + def at(events, action, x=None, start=0): + return next(i for i, e in enumerate(events) + if i >= start and e["a"] == action and (x is None or e["x"] == x)) + + def move(events, src, dst): + events.insert(dst, events.pop(src)) + + def shorten_closed(events): + # Shorten a slot that was already answered. + i = at(events, "Shorten") + r = events[i]["r"] + answered = next(j for j, e in enumerate(events) + if e["a"] == "Discharge" and e["r"] == r) + move(events, i, answered) + require_rejected(by_name["replies_shorten"], shorten_closed, + "deadline shortened after the reply", tlc) + + def second_job_shares(events): + # A second job for a request whose first job still runs, with no + # free credit (BOUND = 1): ShareRunning. + events[at(events, "PipeReply", "refused")]["x"] = "linked" + require_rejected(by_name["replies_job_deadline"], second_job_shares, + "second linked job without a credit", tlc) + + def notified_twice(events): + i = at(events, "JobNotify") + events.insert(i + 1, dict(events[i])) + require_rejected(by_name["replies_job_done"], notified_twice, + "job notification taken twice", tlc) + + def end_before_notify(events): + # The job's work ends while its deadline notification is pending. + move(events, at(events, "JobEnd"), at(events, "JobNotify")) + require_rejected(by_name["replies_job_deadline"], end_before_notify, + "job ended before its notification was taken", tlc) + + def cancel_while_waiting(events): + # Cancelled while its caller still waits (no give-up, no exit). + move(events, at(events, "JobCancel"), at(events, "Discharge", "abandoned")) + require_rejected(by_name["replies_job_cancelled"], cancel_while_waiting, + "linked job cancelled while its caller waits", tlc) + + def call_before_job(events): + move(events, at(events, "CallB"), at(events, "PipeReply")) + require_rejected(by_name["replies_downstream_now"], call_before_job, + "downstream request with no running job", tlc) + + def dispatch_without_call(events): + events.pop(at(events, "CallB")) + require_rejected(by_name["replies_downstream_now"], dispatch_without_call, + "B dispatches a request no job made", tlc) + + def reply_before_defer(events): + move(events, at(events, "BReply"), at(events, "BDispatch")) + require_rejected(by_name["replies_downstream_later"], reply_before_defer, + "B answers a request it has not deferred", tlc) + + def stuck_while_running(events): + # The cleanup deadline before exit began. + move(events, at(events, "Stuck"), at(events, "Exit")) + require_rejected(by_name["replies_stuck"], stuck_while_running, + "cleanup deadline before exit", tlc) + + +def lifecycle_trace_controls(traces, tlc): + """Edits of recorded base-lifecycle traces that Lifecycle.tla must + reject: each reintroduces a v0.7 behaviour or breaks a rank 5e/5g rule.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "lifecycle-v1"} + + def spawned(events, n): + return [e["t"] for e in events if e["a"] == "Spawn"][n - 1] + + def index(events, action, x=None, after=0): + return next(i for i, e in enumerate(events) + if i >= after and e["a"] == action and (x is None or e["x"] == x)) + + def hook_after_kill(events): + events[index(events, "Hook", "skip")]["x"] = "run" + require_rejected(by_name["lifecycle_kill_queued_request"], hook_after_kill, + "stopped() runs after a kill", tlc) + + def killed_as_kill(events): + events[index(events, "Die", "Killed")]["x"] = "Kill" + require_rejected(by_name["lifecycle_kill_queued_request"], killed_as_kill, + "a kill ends Kill instead of Killed (v0.7)", tlc) + + def publish_before_death(events): + events.insert(index(events, "Die"), events.pop(index(events, "Publish"))) + require_rejected(by_name["lifecycle_kill_queued_request"], publish_before_death, + "publication before the death point", tlc) + + def untrapped_runs_hook(events): + events[index(events, "Hook", "skip")]["x"] = "run" + require_rejected(by_name["lifecycle_untrapped_signal"], untrapped_runs_hook, + "stopped() runs after an untrapped exit signal (v0.7)", tlc) + + def normal_signal_stops(events): + # v0.7: a Normal signal stopped a non-trapping actor. + at = index(events, "Signal", "Normal") + events[at + 1:at + 1] = [ + {"a": "Hook", "t": events[at]["t"], "s": 0, "x": "skip", "m": 0, "k": "", "o": ""}] + require_rejected(by_name["lifecycle_untrapped_signal"], normal_signal_stops, + "a Normal exit signal stops a non-trapping actor (v0.7)", tlc) + + def down_with_real_reason(events): + down = next(e for e in events if e["a"] == "Deliver" and e["k"] == "down") + down["x"] = "Normal" + require_rejected(by_name["lifecycle_late_registrations"], down_with_real_reason, + "late monitor's Down carries the real reason, not NoProc (v0.7)", tlc) + + def late_link_with_real_reason(events): + notice = next(e for e in events if e["a"] == "Deliver" and e["k"] == "link") + notice["x"] = "Normal" + require_rejected(by_name["lifecycle_late_registrations"], late_link_with_real_reason, + "late link's notice carries the real reason, not NoProc (v0.7)", tlc) + + def notice_after_unlink(events): + a1, a2 = spawned(events, 1), spawned(events, 2) + at = next(i for i, e in enumerate(events) if e["a"] == "Publish" and e["t"] == a2) + events.insert(at + 1, {"a": "Deliver", "t": a1, "s": a2, "x": "Shutdown", + "m": 0, "k": "link", "o": "death"}) + require_rejected(by_name["lifecycle_unlink_fence"], notice_after_unlink, + "death notice delivered after unlink (v0.7 fence)", tlc) + + def down_after_demonitor(events): + a1, a2 = spawned(events, 1), spawned(events, 2) + m = next(e["m"] for e in events if e["a"] == "Monitor") + at = next(i for i, e in enumerate(events) if e["a"] == "Publish" and e["t"] == a2) + events.insert(at + 1, {"a": "Deliver", "t": a1, "s": a2, "x": "Normal", + "m": m, "k": "down", "o": "death"}) + require_rejected(by_name["lifecycle_demonitor_fence"], down_after_demonitor, + "Down delivered after demonitor (v0.7 fence)", tlc) + + def parent_exit_as_exit(events): + a2 = spawned(events, 2) + next(e for e in events if e["a"] == "Dispatch" and e["t"] == a2 + and e["k"] == "pexit")["k"] = "exit" + require_rejected(by_name["lifecycle_parent_exit"], parent_exit_as_exit, + "parent Shutdown reaches exit_received", tlc) + + def drop_keeps_normal(events): + events[index(events, "Drop")]["x"] = "Normal" + require_rejected(by_name["lifecycle_runtime_drop"], drop_keeps_normal, + "runtime drop ends Normal instead of Killed (v0.7)", tlc) + + # Actor-to-actor messages (the model's Send). + def send_after_death(events): + send = index(events, "Send") + die = index(events, "Die", "Shutdown") + events.insert(die, events.pop(send)) + require_rejected(by_name["lifecycle_message_then_exit"], send_after_death, + "message sent by an actor after its death point", tlc) + + def notice_before_message(events): + send = events.pop(index(events, "Send")) + deliver = index(events, "Deliver") + events.insert(deliver + 1, send) + require_rejected(by_name["lifecycle_message_then_exit"], notice_before_message, + "death notice overtakes the dying actor's earlier message", tlc) + + def message_as_request(events): + msg = index(events, "Dispatch") + while events[msg]["k"] != "msg": + msg = index(events, "Dispatch", after=msg + 1) + events[msg]["k"] = "req" + require_rejected(by_name["lifecycle_threads_actor_message"], message_as_request, + "actor-to-actor message dispatched as an outside request", tlc) + + +def job_trace_controls(traces, tlc): + """Edits of recorded pipe-to-self job traces that DeferredJobs.tla must + reject: each erases or reorders what the job core enforces.""" + by_name = {t["name"]: t for t in traces if t.get("schema") == "jobs-v1"} + + def deliver_replaced(events): + next(e for e in events if e["a"] == "Dispatch" and e["fenced"])["fenced"] = False + require_rejected(by_name["jobs_keyed_fenced"], deliver_replaced, + "replaced keyed job's terminal message delivered", tlc) + + def emit_after_terminal(events): + finish = next(i for i, e in enumerate(events) if e["a"] == "Finish") + last_emit = max(i for i, e in enumerate(events) if e["a"] == "Emit") + events.insert(last_emit, events.pop(finish)) + require_rejected(by_name["jobs_emits"], emit_after_terminal, + "emit enqueued after the terminal message", tlc) + + def skip_credit_return(events): + events.remove(next(e for e in events if e["a"] == "Dispatch")) + require_rejected(by_name["jobs_emits"], skip_credit_return, + "second emit with its credit still in flight", tlc) + + def second_terminal(events): + next(e for e in events if e["a"] == "Finish")["x"] = "notified" + require_rejected(by_name["jobs_deadline"], second_terminal, + "terminal message after the job deadline's", tlc) + + def forget_cancel(events): + events.remove(next(e for e in events if e["a"] == "Cancel")) + require_rejected(by_name["jobs_cancelled"], forget_cancel, + "quiet finish of a job that was never cancelled", tlc) + + def dispatch_after_exit(events): + exit_at = next(i for i, e in enumerate(events) if e["a"] == "Exit") + dispatch = next(e for e in events if e["a"] == "Dispatch") + events.remove(dispatch) + events.insert(exit_at, dispatch) + require_rejected(by_name["jobs_plain"], dispatch_after_exit, + "dispatch after the owner's exit", tlc) + + # Delayed emits (emit_at) and held results. + def release_before_fire(events): + release = next(i for i, e in enumerate(events) if e["a"] == "Release") + fire = next(i for i, e in enumerate(events) if e["a"] == "Fire") + events.insert(fire, events.pop(release)) + require_rejected(by_name["jobs_held_release"], release_before_fire, + "held result released before its last delayed emit fired", tlc) + + def terminal_while_scheduled(events): + next(e for e in events if e["a"] == "Finish")["x"] = "notified" + require_rejected(by_name["jobs_held_release"], terminal_while_scheduled, + "terminal message enqueued while a delayed emit is scheduled", tlc) + + def release_without_last_fire(events): + next(e for e in events if e["a"] == "Fire")["rel"] = False + require_rejected(by_name["jobs_held_release"], release_without_last_fire, + "held result released by a fire that did not take it", tlc) + + def emit_after_deadline(events): + next(e for e in events if e["a"] == "Fire")["x"] = "emit" + require_rejected(by_name["jobs_held_deadline"], emit_after_deadline, + "delayed emit enqueued after the job deadline took the result", tlc) + + def fire_unscheduled(events): + events.remove(next(e for e in events if e["a"] == "Schedule")) + require_rejected(by_name["jobs_due_now"], fire_unscheduled, + "delayed emit fired without being scheduled", tlc) + + def drop_before_cancel(events): + cancel = next(i for i, e in enumerate(events) if e["a"] == "Cancel") + fire = next(i for i, e in enumerate(events) if e["a"] == "Fire") + events.insert(cancel, events.pop(fire)) + require_rejected(by_name["jobs_cancel_scheduled"], drop_before_cancel, + "delayed emit dropped while the job still accepts it", tlc) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("traces", type=Path) @@ -339,13 +676,15 @@ def erase_drain_failure(events): event["x"] = "Normal" require_rejected(drain, erase_drain_failure, "transient restart after erased queued destructor panic", args.tlc) - shutdown = next(t for t in traces if t["name"] == "queued_drop_preserves_shutdown") - def escalate_shutdown(events): + shutdown = next(t for t in traces if t["name"] == "queued_drop_shutdown_restarts") + def drop_cleanup_failure(events): + # v0.8: a cleanup failure after Shutdown projects to the abnormal + # class; projecting the raw Shutdown instead must reject the restart. for event in events: - if event.get("x") == "Shutdown": - event["x"] = "Panic" - require_rejected(shutdown, escalate_shutdown, - "transient retirement after incorrectly escalated shutdown", args.tlc) + if event.get("a") == "SelfExit" and event.get("x") == "Panic": + event["x"] = "Shutdown" + require_rejected(shutdown, drop_cleanup_failure, + "transient restart after a Shutdown without its cleanup failure", args.tlc) replacement = next(t for t in traces if t["name"] == "transient_replacement_normal_before_ready") def fail_instead_of_retry(events): failed = next(i for i, e in enumerate(events) @@ -365,12 +704,37 @@ def fail_instead_of_retry(events): model_fault("MCCohortsAll", fault, args.tlc) # Needs a second generation within one round (MaxGen 3). model_fault("MCCohortsAllSupersession", "InPlaceRetry", args.tlc) + model_fault("MCCohortsAll", "FirstStartRetry", args.tlc, invariant="Safety") + # Rank 5f: v0.7 charged every rebuilt child of a round. + named_fault("MCCohortsAll", "PerChildCharge", "RoundCharged", args.tlc) + for label, pattern, replacement in SUPERVISION_S1_CONTROLS: + spec_mutation("MCSupervision", "Supervision.tla", label, pattern, replacement, + "RestartFollowsPolicy", args.tlc) + deferred = next(t for t in traces if t["name"] == "ready_exit_deferred") + def decide_at_once(events): + # v0.7: no deferral; the restart decision is taken during startup, + # where the deferral was recorded (before SupStartDone). + at = next(i for i, e in enumerate(events) if e["a"] == "DeferExit") + child = events[at]["c"] + undefer = next(i for i, e in enumerate(events) if e["a"] == "Undefer") + decide = next(i for i, e in enumerate(events) + if i > undefer and e["a"] == "Decide" and e["c"] == child) + events[at] = events.pop(decide) + events.pop(undefer) + require_rejected(deferred, decide_at_once, + "ready child's exit decided during the initial start", args.tlc) model_fault("MCCoordinator", "ReleaseEarly", args.tlc) model_fault("MCCoordinator", "ForgetFailure", args.tlc) model_fault("MCCoordinator", "ForgetSignal", args.tlc, invariant="JoinAfterAllSignalled") + # A root left unsignalled is also left unkilled at grace expiry: drop + # GraceKillsRemaining too, so the run reaches the liveness check. model_fault("MCCoordinator", "ForgetSignal", args.tlc, temporal=True, - drop=("JoinAfterAllSignalled",)) + drop=("JoinAfterAllSignalled", "GraceKillsRemaining")) model_fault("MCCoordinator", "RollbackReady", args.tlc, invariant="OrderedShutdown") + # v0.8 grace expiry: no kill (a stuck trapping root holds the join), or a + # kill in the ordered loop (stopped() skipped before the grace expired). + model_fault("MCCoordinator", "NoKillAtGrace", args.tlc, invariant="GraceKillsRemaining") + model_fault("MCCoordinator", "KillEarly", args.tlc, invariant="KillOnlyAtGrace") model_fault("MCNested", "Join", args.tlc) # A Temporary child restarted after a panic (MCSupervision has no # Temporary child, so only MCSupervisionTemporary can catch this). @@ -380,6 +744,9 @@ def fail_instead_of_retry(events): model_fault("MCNested", "Propagation", args.tlc, temporal=True) for module, fault, invariant in DEFERRED_CONTROLS: named_fault(module, fault, invariant, args.tlc) + job_trace_controls(traces, args.tlc) + reply_trace_controls(traces, args.tlc) + lifecycle_trace_controls(traces, args.tlc) for module, fault, invariant in LIFECYCLE_CONTROLS: named_fault(module, fault, invariant, args.tlc) for module, fault, invariant in INTROSPECTION_CONTROLS: diff --git a/verification/tla/trace_check.py b/verification/tla/trace_check.py index d39a40aec..da2eaa03a 100644 --- a/verification/tla/trace_check.py +++ b/verification/tla/trace_check.py @@ -1,5 +1,7 @@ #!/usr/bin/env python3 -"""Check recorded supervisor lifecycle traces against Supervision.tla. +"""Check recorded lifecycle traces against Supervision.tla (and, by schema, +Cohorts.tla, NestedSupervision.tla, DeferredJobs.tla, DeferredReplies.tla and +Lifecycle.tla). Each JSON trace (written by concurrency/tests/supervision_trace.rs with the `lifecycle-trace` feature) looks like: @@ -47,10 +49,17 @@ def tla_event(event, cohort=False): def generate(trace, workdir): schema = trace.get("schema", "one-for-one-v1") - if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1"}: + if schema not in {"one-for-one-v1", "cohort-v1", "tree-v1", "jobs-v1", "replies-v1", + "lifecycle-v1"}: raise ValueError(f"unsupported trace schema: {schema}") if schema == "tree-v1": return generate_tree(trace, workdir) + if schema == "jobs-v1": + return generate_jobs(trace, workdir) + if schema == "replies-v1": + return generate_replies(trace, workdir) + if schema == "lifecycle-v1": + return generate_lifecycle(trace, workdir) cohort = schema == "cohort-v1" modules = ("Cohorts.tla", "TraceCohorts.tla") if cohort else MODELS for model in modules: @@ -106,6 +115,207 @@ def generate_tree(trace, workdir): "INVARIANT Safety\nINVARIANT TraceIncomplete\n") +def generate_replies(trace, workdir): + """One owner instance's reply slots against DeferredReplies.tla.""" + for model in ("DeferredReplies.tla", "TraceReplies.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + # A slot address can be reused once freed: each Send starts a request. + current, n, records = {}, 0, [] + for e in trace["events"]: + r = int(e["r"]) + if e["a"] == "Send": + n += 1 + current[r] = n + req = current.get(r, 0) if r else 0 + if r and not req: + raise ValueError(f"event for an unsent request: {e}") + records.append("[a |-> {}, r |-> {}, x |-> {}]".format( + tla_string(e["a"]), req, tla_string(e["x"]))) + if not n: + raise ValueError("a replies trace needs at least one request") + # Deadlines are chosen by TLC; the clock only needs room for the trace. + clock = 2 + sum(1 for e in trace["events"] if e.get("x") == "timeout") + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceReplies\nTraceLog == <<\n " + + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" N = {n}\n BoundA = {int(trace['bound'])}\n" + f" BoundB = {int(trace.get('bound_b', 1))}\n" + f" MaxTime = {clock}\n MaxWait = {clock}\n Fault = \"None\"\n" + " Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + +OWN_ACTIONS = {"Trap", "Stop", "Link", "Unlink", "Monitor", "Demonitor"} + + +def lifecycle_records(trace): + """Normalize a lifecycle-v1 trace: actor and monitor numbers, Send split + into Request and Enqueue, and request turns that panicked folded into + their Dispatch (TraceLifecycle.tla). Returns (records, constants).""" + events = trace["events"] + actors, parents = {}, {} + for e in events: + if e["a"] == "Spawn": + actors[int(e["t"])] = f"a{len(actors) + 1}" + if int(e["s"]): + parents[int(e["t"])] = int(e["s"]) + if not actors: + raise ValueError("a lifecycle trace needs at least one spawned actor") + for child, parent in parents.items(): + if parent not in actors: + raise ValueError(f"start_linked parent {parent} is not recorded") + + def actor(raw): + if int(raw) not in actors: + raise ValueError(f"event for an unrecorded actor {raw}") + return actors[int(raw)] + + # Request turns that panicked with nothing of the actor's own in between. + folded = set() + turn = {} + for i, e in enumerate(events): + t = int(e["t"]) + if e["a"] == "Dispatch": + turn[t] = i if e["k"] == "req" else None + elif e["a"] in OWN_ACTIONS: + turn[t] = None + elif e["a"] == "Panic" and e["x"] == "turn" and turn.get(t) is not None: + folded |= {turn[t], i} + turn[t] = None + + monitors, requests, children, sends = {}, 0, set(), 0 + signals = late = 0 + records = [] + + def record(a, t="none", s="none", x="", m=0, q=0, k="", o="", p=False): + records.append("[a |-> {}, t |-> {}, s |-> {}, x |-> {}, m |-> {}, q |-> {}, k |-> {}, o |-> {}, p |-> {}]".format( + tla_string(a), tla_string(t), tla_string(s), tla_string(x), m, q, + tla_string(k), tla_string(o), "TRUE" if p else "FALSE")) + + for i, e in enumerate(events): + a = e["a"] + if a == "Spawn": + continue + t = actor(e["t"]) + if a == "Send": + # A plain message from a recorded actor: the model's Send(s, t). + if int(e.get("s", 0)) and e["x"] == "queued": + sends += 1 + record("Send", t=t, s=actor(e["s"])) + continue + requests += 1 + record("Request", t=t, q=requests) + record("Enqueue", t=t, q=requests, x=e["x"]) + continue + m = 0 + if int(e.get("m", 0)): + m = monitors.setdefault(int(e["m"]), len(monitors) + 1) + if a == "Signal": + if e["o"] == "sup": + source = "sup" + children.add(t) + else: + source = actors.get(int(e["s"]), "ext") + signals += 1 + record(a, t=t, s=source, x=e["x"]) + continue + if a == "Stop": + signals += 1 + if a in {"Link", "Monitor"} and e["x"] in {"noproc", "try", "late"}: + late += 1 + source = actor(e["s"]) if a in {"Link", "Unlink", "Monitor", "Deliver"} else "none" + record(a, t=t, s=source, x=e["x"], m=m, k=e["k"], o=e["o"], p=i in folded) + constants = { + "actors": [actors[k] for k in actors], + "parents": {actors[c]: actors[p] for c, p in parents.items()}, + "children": sorted(children), + "requests": requests, + "monitors": len(monitors), + "signals": signals, + "late": late, + "sends": sends, + } + return records, constants + + +def generate_lifecycle(trace, workdir): + """A group of actors' exit signals, links, monitors and requests against + Lifecycle.tla.""" + for model in ("Lifecycle.tla", "TraceLifecycle.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + records, c = lifecycle_records(trace) + + def tla_set(values): + return "{" + ", ".join(tla_string(v) for v in values) + "}" + if c["parents"]: + parent = "CASE " + " [] ".join(f"a = {tla_string(ch)} -> {tla_string(p)}" + for ch, p in c["parents"].items()) + ' [] OTHER -> "none"' + else: + parent = '"none"' + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceLifecycle\n" + f"TraceActors == {tla_set(c['actors'])}\n" + f"TraceChildren == {tla_set(c['children'])}\n" + f"TraceLinkParent == [a \\in TraceActors |-> {parent}]\n" + f"TraceReqs == 1..{c['requests']}\n" + f"TraceMonIds == 1..{c['monitors']}\n" + "TraceLog == <<\n " + ",\n ".join(records) + "\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + " Actors <- TraceActors\n Children <- TraceChildren\n" + " LinkParent <- TraceLinkParent\n Reqs <- TraceReqs\n" + f" MonIds <- TraceMonIds\n MaxSends = {c['sends']}\n" + f" MaxSignals = {c['signals']}\n MaxLate = {c['late']}\n" + " UnlinkEnabled = TRUE\n TrapCanClear = TRUE\n" + " SelfReasons <- Reasons\n SignalReasons <- Reasons\n" + " AdminReasons <- Reasons\n DropEnabled = TRUE\n" + " Fault = \"None\"\n Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + +def generate_jobs(trace, workdir): + """One owner instance's pipe-to-self jobs against DeferredJobs.tla.""" + for model in ("DeferredJobs.tla", "TraceJobs.tla"): + shutil.copy(os.path.join(HERE, model), workdir) + if int(trace.get("keys", 0)) > 1: + raise ValueError("DeferredJobs.tla models one key per trace") + # Number jobs 1..J in the order they appear; Exit carries job 0. + jobs = {} + for e in trace["events"]: + if int(e["j"]): + jobs.setdefault(int(e["j"]), len(jobs) + 1) + if not jobs: + raise ValueError("a jobs trace needs at least one job") + keyed = sorted({jobs[int(e["j"])] for e in trace["events"] if e["a"] == "Start" and e["keyed"]}) + # Emits and delayed emits (Schedule) both count towards MaxEmit. + emits = {} + for e in trace["events"]: + if e["a"] in {"Emit", "Schedule"}: + emits[int(e["j"])] = emits.get(int(e["j"]), 0) + 1 + + def flag(value): + return "TRUE" if value else "FALSE" + + records = ",\n ".join( + "[a |-> {}, j |-> {}, x |-> {}, keyed |-> {}, dl |-> {}, fenced |-> {}, rel |-> {}]".format( + tla_string(e["a"]), jobs.get(int(e["j"]), 0), tla_string(e["x"]), + flag(e["keyed"]), flag(e["dl"]), flag(e["fenced"]), flag(e.get("rel", False))) + for e in trace["events"]) + with open(os.path.join(workdir, "TraceRun.tla"), "w") as out: + out.write("---- MODULE TraceRun ----\nEXTENDS TraceJobs\n" + "TraceKeyed == {" + ", ".join(map(str, keyed)) + "}\n" + f"TraceLog == <<\n {records}\n>>\n====\n") + with open(os.path.join(workdir, "TraceRun.cfg"), "w") as out: + out.write("INIT TraceInit\nNEXT TraceNext\nCONSTANTS\n" + f" J = {len(jobs)}\n JobBound = {len(jobs)}\n MaxGen = 1\n" + f" MaxEmit = {max(emits.values(), default=0)}\n" + f" Credits = {int(trace['credits'])}\n Fault = \"None\"\n" + " Keyed <- TraceKeyed\n Trace <- TraceLog\n" + "INVARIANT Safety\nINVARIANT TraceIncomplete\n") + + def longest_prefix(dump_path): best = 0 try: diff --git a/verification/tla/trace_coverage.py b/verification/tla/trace_coverage.py index 42994616e..0338170b7 100644 --- a/verification/tla/trace_coverage.py +++ b/verification/tla/trace_coverage.py @@ -32,6 +32,13 @@ [--model-keys CACHE] [--json OUT] [--baseline FILE] [--write-baseline FILE] [--top N] +Lifecycle.tla, DeferredJobs.tla and DeferredReplies.tla are covered by +action keys only ("static" models): the model side is the set of log +event arms of their trace spec (TraceLifecycle.tla, TraceJobs.tla, +TraceReplies.tla; each arm is one model action or a named group of them), +and a trace covers the arms its accepted replay takes. No state space is +explored for them, so they add no TLC run to the model side. + Each --traces source is reported cumulatively in the given order (for example the dedicated trace tests, then the whole test suite). --baseline FILE fails (exit 1) if an action key listed there is no longer covered; coverage @@ -84,6 +91,26 @@ }, } +# Models covered by their trace spec's arms (action keys only): name -> +# (trace spec, schemas). See the module docstring. +STATIC = { + "Lifecycle": ("TraceLifecycle.tla", {"lifecycle-v1"}), + "DeferredJobs": ("TraceJobs.tla", {"jobs-v1"}), + "DeferredReplies": ("TraceReplies.tla", {"replies-v1"}), +} +ALL = list(MODELS) + list(STATIC) + + +def static_model_keys(): + """The arms (`e.a = "X"`) of each static model's trace spec.""" + keys = {} + for name, (spec, _) in STATIC.items(): + with open(os.path.join(HERE, spec)) as f: + arms = re.findall(r'e\.a = "(\w+)"', f.read()) + keys[name] = {"A|" + a for a in arms} + return keys + + # Steps the model takes that no trace event can name (fairness devices). UNOBSERVABLE = {"Nested": ("A|Deliver", "T|Deliver|")} @@ -244,9 +271,30 @@ def model_of(trace): for name, spec in MODELS.items(): if schema in spec["schemas"]: return name + for name, (_, schemas) in STATIC.items(): + if schema in schemas: + return name raise ValueError(f"unsupported trace schema: {schema}") +def static_trace_keys(trace, model, tlc): + """Replay TRACE with trace_check's TraceRun and return the arms its log + takes (only if TLC accepts it).""" + workdir = tempfile.mkdtemp(prefix="cov-trace-") + try: + trace_check.generate(trace, workdir) + cmd = shlex.split(tlc) + ["-deadlock", "-workers", "1", "-cleanup", + "-metadir", os.path.join(workdir, "states"), "TraceRun"] + run = subprocess.run(cmd, cwd=workdir, env=tlc_env(workdir), capture_output=True, text=True) + if "Invariant TraceIncomplete is violated" not in run.stdout + run.stderr: + return model, "REJECTED", set() + with open(os.path.join(workdir, "TraceRun.tla")) as f: + arms = set(re.findall(r'\ba \|-> "(\w+)"', f.read())) + return model, "ACCEPTED", {"A|" + a for a in arms} + finally: + shutil.rmtree(workdir, ignore_errors=True) + + COV_TRACE = {"Supervision": "CovTraceSupervision", "Cohorts": "CovTraceCohorts", "Nested": "CovTraceNested"} @@ -258,6 +306,8 @@ def trace_keys(path, tlc): model = model_of(trace) if not trace.get("events"): return model, "EMPTY", set() + if model in STATIC: + return static_trace_keys(trace, model, tlc) workdir = tempfile.mkdtemp(prefix="cov-trace-") try: trace_check.generate(trace, workdir) @@ -288,6 +338,17 @@ def trace_keys(path, tlc): shutil.rmtree(workdir, ignore_errors=True) +# Trace schemas validated by trace_check.py but without a coverage model: +# skipped here (none now: the deferred-replies models and Lifecycle.tla are +# STATIC models). +UNCOVERED_SCHEMAS = set() + + +def covered(path): + with open(path) as f: + return json.load(f).get("schema") not in UNCOVERED_SCHEMAS + + def collect(paths): files = [] for p in paths: @@ -295,7 +356,7 @@ def collect(paths): files += sorted(os.path.join(p, f) for f in os.listdir(p) if f.endswith(".json")) elif os.path.exists(p): files.append(p) - return files + return [f for f in files if covered(f)] # --------------------------------------------------------------------------- @@ -331,6 +392,7 @@ def collect(paths): "Supervision": {"Permanent", "Transient", "Temporary", "failed", "0", "mid", "max", "reserved"}, "Cohorts": {"Permanent", "Transient", "Temporary", "before", "after", "origin", "none"}, "Nested": set(), + **{name: set() for name in STATIC}, } @@ -401,18 +463,19 @@ def main(): for name, prefixes in UNOBSERVABLE.items(): model[name] = {k for k in model[name] if not k.startswith(prefixes)} model = {name: with_coarse(name, keys) for name, keys in model.items()} + model.update(static_model_keys()) sources = [] for spec in args.traces: label, _, path = spec.rpartition("=") sources.append((label or path, collect([path]))) - covered = {m: set() for m in MODELS} + covered = {m: set() for m in ALL} verdicts = [] rows = [] with concurrent.futures.ThreadPoolExecutor(args.jobs) as pool: for label, files in sources: - counts = {m: [0, 0] for m in MODELS} + counts = {m: [0, 0] for m in ALL} for path, (name, verdict, keys) in zip(files, pool.map(lambda f: trace_keys(f, args.tlc), files)): verdicts.append({"source": label, "trace": os.path.basename(path), "model": name, "verdict": verdict, "keys": len(keys)}) @@ -422,7 +485,7 @@ def main(): covered[name] |= with_coarse(name, keys) elif verdict == "REJECTED": print(f"not counted (rejected by the model): {label}: {os.path.basename(path)}") - for name in MODELS: + for name in ALL: mk, tk = kinds(model[name]), kinds(covered[name] & model[name]) rows.append((label, name, counts[name], {k: (len(tk[k]), len(mk[k])) for k in KINDS})) @@ -436,8 +499,10 @@ def main(): beyond = covered[name] - model[name] print(f"{name}: model {state_counts[name]} states over {len(MODELS[name]['configs'])} configuration(s); " f"{len(beyond)} trace keys outside the configured bounds") + for name in STATIC: + print(f"{name}: action keys are the arms of {STATIC[name][0]}") - for name in MODELS: + for name in ALL: # Rank unreached actions and coarse transitions; each coarse key # stands for its unreached fine variants (count shown). missing = model[name] - covered[name] @@ -457,12 +522,12 @@ def main(): json.dump({ "model": {m: sorted(v) for m, v in model.items()}, "covered": {m: sorted(v) for m, v in covered.items()}, - "unreached": {m: sorted(model[m] - covered[m], key=lambda k: (-score(m, k), k)) for m in MODELS}, - "beyond": {m: sorted(covered[m] - model[m]) for m in MODELS}, + "unreached": {m: sorted(model[m] - covered[m], key=lambda k: (-score(m, k), k)) for m in ALL}, + "beyond": {m: sorted(covered[m] - model[m]) for m in ALL}, "traces": verdicts, }, f, indent=1) - actions = {m: sorted(k for k in covered[m] & model[m] if k.startswith("A|")) for m in MODELS} + actions = {m: sorted(k for k in covered[m] & model[m] if k.startswith("A|")) for m in ALL} if args.write_baseline: with open(args.write_baseline, "w") as f: json.dump(actions, f, indent=1)