From 749a61a851a5c7a5aa18ba51cfcc80d8da7b2e62 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 01:12:45 +0530 Subject: [PATCH 01/20] feat(harness): hosted world handle over per-world postgres stores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 of the hosted execution guest: the typed world surface (state/put/change/drop/call/query) scenario code receives, backed by a per-world logical database — baseline-measured state caps, read-only sub-handles, SQL statement guards, typed errors, and offline + docker test lanes. call() raises until the http_tool shim wire format is pinned (recorded contract defect). Known defects tracked in the implementation ledger. Signed-off-by: khushalsonawat --- src/fi/alk/harness/world/errors.py | 68 ++ src/fi/alk/harness/world/handle.py | 493 ++++++++++++++ src/fi/alk/harness/world/runtime.py | 2 +- src/fi/alk/harness/world/stores/__init__.py | 11 +- src/fi/alk/harness/world/stores/postgres.py | 146 +++- tests/harness/test_world_handle.py | 701 ++++++++++++++++++++ 6 files changed, 1387 insertions(+), 34 deletions(-) create mode 100644 src/fi/alk/harness/world/errors.py create mode 100644 src/fi/alk/harness/world/handle.py create mode 100644 tests/harness/test_world_handle.py diff --git a/src/fi/alk/harness/world/errors.py b/src/fi/alk/harness/world/errors.py new file mode 100644 index 00000000..5b1a9367 --- /dev/null +++ b/src/fi/alk/harness/world/errors.py @@ -0,0 +1,68 @@ +"""What the hosted world handle raises when scenario code asks it for something it will not do. + +Every one of these is scenario code at fault, never the world's contents and never the agent +under test: a `KeyError` from a missing table reads as a finding about data, and a bare +`StoreError` from the postgres driver reads as an infrastructure fault. Neither is right for +"you called `change` without saying which column `key` names," so the handle has its own +vocabulary, and folds it under one base class for whoever has to route "scenario code misused +the handle" to one outcome without naming all six. +""" + +from __future__ import annotations + + +class WorldError(RuntimeError): + """Scenario code asked the world handle for something it will not do.""" + + +class WorldReadOnly(WorldError): + """`put`, `change`, `drop` or `call` reached the handle `ready()` or `check()` were given. + + Those two only ever observe a run. A check that could write would be able to change the very + thing it is grading, and nothing downstream could tell the difference between a check that + found a problem and one that quietly fixed it. + """ + + +class WorldReservedName(WorldError): + """Scenario code named the harness's own conformance canary. + + That table exists to prove worlds are really isolated from each other, not to hold scenario + data, and it never appears in `state()` either. + """ + + +class WorldQueryRejected(WorldError): + """`query()` was handed something that is not one plain read. + + The database's own read-only transaction is what actually stops a write; this is the + friendlier rejection in front of it, so a statement that was never going to be allowed fails + on a message naming the reason rather than a lock error three layers down. + """ + + +class WorldStateTooLarge(WorldError): + """`state()` reached a table whose row count, measured when the baseline was frozen, passed + the cap. + + Measured once, at freeze time, by the provisioner — never recomputed here, so which tables + raise is fixed before a scenario ever runs and nothing a call does during one can move it. + """ + + +class WorldUnavailable(WorldError): + """The handle cannot do this, given how the world in front of it is built — not what it holds. + + A postgres world whose `public` schema has no tables, and `call()` — which raises + unconditionally until the `http_tool` shim's wire format is pinned somewhere in the contracts + — are both this: nothing went wrong, the capability was never there. + """ + + +class WorldUsageError(WorldError): + """A `put`, `change` or `drop` cannot be carried out as asked. + + Inserting into something that is not a table, or changing or dropping a record without + saying which column `key` names — hosted worlds cannot invent tables or guess a column, so + both are reported here rather than attempted. + """ diff --git a/src/fi/alk/harness/world/handle.py b/src/fi/alk/harness/world/handle.py new file mode 100644 index 00000000..58481bee --- /dev/null +++ b/src/fi/alk/harness/world/handle.py @@ -0,0 +1,493 @@ +"""The hosted world handle: the shipped world vocabulary, backed by one world's own postgres. + +`GeneratedWorld` is a database an agent's own generated handlers reach through `Db`. A hosted +world has no handlers to generate — the tables are whatever the agent's own migrations made, and +what a scenario needs is the same six-verb surface (`state`, `put`, `change`, `drop`, `call`, +`query`) built directly on the store, with nothing to adopt or reimplement per agent. Everything +this module refuses, it refuses before the database sees it, so a scenario's mistake reads as a +message naming what it did wrong rather than a `KeyError` or a driver traceback three layers down. +""" + +from __future__ import annotations + +import random +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from .errors import ( + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from .runtime import Call +from .stores.postgres import PostgresStore + +# The harness's own isolation canary. It exists to prove worlds are really separate from each +# other, never to hold scenario data, so scenario code is never allowed to see or touch it. +CONFORMANCE_TABLE = "_alk_conformance" + +# Measured once, at baseline freeze, by the provisioner — never recomputed here. A table over +# this stays over it for the whole run; nothing a scenario does can move which tables raise. +STATE_ROW_CAP = 5000 + +# The only statement shapes `query()` accepts. Anything else is refused before it reaches the +# database's own read-only transaction, so a statement that was never going to be allowed fails +# on a message naming why rather than a lock error from three layers down. +_READ_KEYWORDS = {"select", "with", "values"} + +# The read-only view's fallback answer is restricted to exactly this vocabulary, so a genuinely +# unknown attribute — a capability probe, a dunder, a typo — still reads as a plain +# `AttributeError` instead of masquerading as a write refusal. +_WRITE_VERBS = frozenset({"put", "change", "drop", "call"}) + + +class HostedWorld: + """The `World` surface, backed by this scenario's own logical postgres database. + + One handle per scenario, over one short-lived autocommit connection per operation — nothing + held open, which is what lets `reset` drop the database out from under a discarded world. + `world_index` and `rng` are plain data: the index is for diagnostics only, and the generator + is the only sanctioned source of randomness scenario code may use. + """ + + def __init__( + self, + store: PostgresStore, + world_index: int, + rng: random.Random, + baseline_row_counts: Mapping[str, int], + ) -> None: + """Wrap `store` as the `World` surface for one scenario. + + `baseline_row_counts` is keyed by the bare `pg_tables.tablename` value — no schema + prefix, no quoting — for every table `public` held when the baseline was frozen. A + visible table missing from this map has no measured cap to enforce, so the coverage + check below fails construction outright rather than waiting for a scenario's first + access to discover a provisioning gap through its own retry. + """ + self._store = store + self.world_index = world_index + self.rng = rng + # Row counts as they stood when the baseline was frozen, keyed by table. Never + # re-measured: a live count would make the cap depend on what a scenario already wrote, + # and the whole point is that it is decided before any scenario runs. + self._baseline_row_counts = dict(baseline_row_counts) + self._require_baseline_coverage(self._visible_tables()) + + # -- reading ------------------------------------------------------------------------------ + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + """A snapshot of the public schema, or of one table in it. + + Bare `state()` leaves an over-cap table out of the snapshot rather than raising through + it — one seeded audit table must not make the primary read verb inert for the whole + run. Naming that table explicitly (`state("big_table")`) still raises: the exclusion is + a property of the snapshot, not a way to read the table around its own cap. If every + visible table is over cap the exclusion would leave the snapshot `{}` — the one thing + state() must never return, since an empty snapshot reads as an observation and makes a + negative check pass on a world nobody actually looked at — so that case raises too, + naming the tables it would have excluded. The exclusion itself happens at the read: the + store is asked for only the included tables, not for every table with the excluded ones + thrown away afterward, so one huge seeded table can no longer make every bare `state()` + pay to materialise and discard rows nobody asked to see. + """ + self._reject_reserved(table) + if table is not None: + visible = self._visible_tables() + self._require_nonempty_schema(visible) + if table not in visible: + raise WorldUsageError( + f"{table!r} is not a table in this world; it holds {sorted(visible)}." + ) + if self._baseline_row_counts[table] > STATE_ROW_CAP: + raise WorldStateTooLarge( + f"{table!r} held {self._baseline_row_counts[table]} rows when the baseline " + f"was frozen, over the {STATE_ROW_CAP}-row cap; state() will not read it " + "back." + ) + return {table: self._store.table(table)} + + names = self._visible_tables() + self._require_nonempty_schema(names) + included = [name for name in names if self._baseline_row_counts[name] <= STATE_ROW_CAP] + if not included: + raise WorldStateTooLarge( + f"every table this world holds — {sorted(names)} — is over the " + f"{STATE_ROW_CAP}-row cap; state() will not return {{}} in their place." + ) + # One connection, asked for only the included tables — a bare state() used to open a + # fresh connection per table (and a second one just to look up its primary key) to read + # every table including the over-cap ones, then throw the over-cap rows away here. + return self._store.state(only=included) + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + """The read escape hatch: one statement, on a transaction that cannot write. + + The database's own read-only transaction is the actual guard; this token check only + makes the common mistake — a stray write, a second statement — fail with a reason + attached instead of a lock error from underneath. + """ + _reject_unless_read(sql) + return self._store.query(sql, tuple(params)) + + # -- writing ------------------------------------------------------------------------------ + + def put(self, collection: str, record: Mapping[str, Any], *, key: str = "") -> dict[str, Any]: + """Insert one record; return exactly what the table stored, generated key included. + + `key` exists only to keep this signature a superset of `GeneratedWorld.put`; a hosted + table already knows its own key — the column its own migrations gave it — so a + scenario naming one here would be telling the table what to call a value the table is + about to generate itself. + """ + self._reject_reserved(collection) + if key: + raise WorldUsageError("a hosted table's key is the table's own — do not pass one.") + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so put() only reaches what the agent's own migrations made." + ) + return self._store.add(collection, dict(record)) + + def change( + self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" + ) -> int: + """Update matching records; return how many changed.""" + self._reject_reserved(collection) + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so change() only reaches what the agent's own migrations made." + ) + by = by or self._resolve_by(collection) + if by not in self._table_columns(collection): + raise WorldUsageError( + f"change({collection!r}, {key!r}, ...) was given by={by!r}, which is not a " + f"column of {collection!r}." + ) + return self._store.amend(collection, key, dict(changes), by=by) + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + """Delete matching records, or every record when `key` is empty; return the count.""" + self._reject_reserved(collection) + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so drop() only reaches what the agent's own migrations made." + ) + if key: + by = by or self._resolve_by(collection) + if by not in self._table_columns(collection): + raise WorldUsageError( + f"drop({collection!r}, {key!r}) was given by={by!r}, which is not a " + f"column of {collection!r}." + ) + return self._store.remove(collection, key, by=by) + + def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: + """Play one of the agent's own tools against this world. + + Not implemented: the `http_tool` evidence seam's wire format is not pinned anywhere in + the contracts yet, and guessing at a shape here would ship one nobody agreed to and + scenario code would end up depending on. Raising unconditionally is the honest answer + until the evidence layer pins it — run-time scenario code should not need this anyway, + since `setup` already runs at proof time and the data verbs cover the rest. + """ + raise WorldUnavailable( + "the http_tool shim wire format is not yet pinned by the contracts — report, " + "don't guess." + ) + + def read_only(self) -> "ReadOnlyWorld": + """The view `ready` and `check` run against: every write verb refuses outright. + + A check able to write could not be told apart from one that quietly repaired what it was + supposed to be grading, so this is what stands between those two functions and the real + handle. + """ + return ReadOnlyWorld(self) + + # -- internal ----------------------------------------------------------------------------- + + def _reject_reserved(self, collection: str | None) -> None: + if collection == CONFORMANCE_TABLE: + raise WorldReservedName( + f"{collection!r} is the harness's own conformance canary, not scenario data; " + "it never appears to scenario code." + ) + + def _require_nonempty_schema(self, visible: list[str]) -> None: + if not visible: + raise WorldUnavailable( + "this world's public schema holds no tables to observe; a postgres world with " + "nothing in it is not a world state() can honestly report on." + ) + + def _require_baseline_coverage(self, names: list[str]) -> None: + """Refuse rather than assume a table the baseline never measured is under the cap. + + A missing entry used to default to a row count of 0, which would let a table nobody + measured at freeze read back as though it were known to be small; failing loud here is + the whole point of deciding the cap before any scenario runs instead of guessing at it. + Called once, from `__init__`: the table set and the baseline are both fixed for the + life of the handle, so checking again on every later access would only repeat a answer + construction already gave. + """ + missing = [name for name in names if name not in self._baseline_row_counts] + if missing: + raise WorldUnavailable( + f"the baseline row counts this world was built with never measured " + f"{sorted(missing)}; state()'s cap cannot be decided for a table nobody " + "measured at freeze." + ) + + def _visible_tables(self) -> list[str]: + rows = self._store.query( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename" + ) + return [row["tablename"] for row in rows if row["tablename"] != CONFORMANCE_TABLE] + + def _table_columns(self, table: str) -> set[str]: + rows = self._store.query( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s", + (table,), + ) + return {row["column_name"] for row in rows} + + def _primary_key_order(self, table: str) -> list[str]: + """This table's primary key columns, in index order. + + Joined through `pg_class`/`pg_namespace` rather than a `%s::regclass` cast over an + f-string, so a table name is only ever a bound value — an embedded `"` (a table created + as `CREATE TABLE "we""ird" (...)`) is just a character in that value instead of + something a regclass cast has to parse. + """ + rows = self._store.query( + """ + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = %s AND i.indisprimary + ORDER BY array_position(i.indkey, a.attnum) + """, + (table,), + ) + return [row["attname"] for row in rows] + + def _resolve_by(self, table: str) -> str: + """The column to key `change`/`drop` on when scenario code did not name one. + + A single-column primary key is the only case unambiguous enough to guess; no primary + key at all, or a composite one, means the store genuinely cannot tell which column + `key` names, so the scenario has to say. + """ + columns = self._primary_key_order(table) + if len(columns) == 1: + return columns[0] + raise WorldUsageError( + f"change/drop on {table!r} needs by=; it has no single-column primary " + "key to default to." + ) + + +class ReadOnlyWorld: + """A `World` whose write verbs are refused before they reach `HostedWorld` at all.""" + + def __init__(self, world: HostedWorld) -> None: + # Name-mangled so code outside this class reaching for `._world` cannot casually + # recover the writable handle a "read-only" view exists to stand in front of. + self.__world = world + self.world_index = world.world_index + self.rng = world.rng + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return self.__world.state(table) + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + return self.__world.query(sql, params) + + def put(self, collection: str, record: Mapping[str, Any], *, key: str = "") -> dict[str, Any]: + raise WorldReadOnly( + f"put({collection!r}, ...) reached a read-only handle; ready() and check() only " + "ever observe a run." + ) + + def change( + self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" + ) -> int: + raise WorldReadOnly( + f"change({collection!r}, {key!r}, ...) reached a read-only handle; ready() and " + "check() only ever observe a run." + ) + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + raise WorldReadOnly( + f"drop({collection!r}, {key!r}) reached a read-only handle; ready() and check() " + "only ever observe a run." + ) + + def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: + raise WorldReadOnly( + f"call({name!r}, ...) reached a read-only handle; ready() and check() only ever " + "observe a run." + ) + + def __getattr__(self, name: str) -> Any: + """A write verb `HostedWorld` grows later, with no override here yet, still reads as + `WorldReadOnly` rather than a plain `AttributeError` indistinguishable from a typo. + + Restricted to that known vocabulary and never to dunders: this repo's own runner code + reaches for `hasattr(world, "forward")` and `getattr(world, "runtime_tools", set())` on + world objects, and both only work through the ordinary `AttributeError` those tools + expect from a name that is simply not there, not from a refusal that happens to look + like one. + """ + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + if name in _WRITE_VERBS: + raise WorldReadOnly( + f"{name!r} reached a read-only handle; ready() and check() only ever observe " + "a run." + ) + raise AttributeError(name) + + +def _is_word_char(char: str) -> bool: + return char.isalnum() or char == "_" + + +_DOLLAR_TAG = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)?\$") + + +def _dollar_quote_end(sql: str, start: int) -> int | None: + """The index just past the matching closing `$tag$`, if `sql[start:]` opens one. + + `None` if `start` is not a dollar-quote opener at all — a `$1` parameter placeholder, or a + bare `$`, never matches the tag grammar and is left for the caller to treat as an ordinary + character. An opener with no matching close consumes to the end of the string, the same + fate an unterminated `'`/`"` span already gets below. + """ + opener = _DOLLAR_TAG.match(sql, start) + if opener is None: + return None + delimiter = opener.group(0) + end = sql.find(delimiter, opener.end()) + return len(sql) if end == -1 else end + len(delimiter) + + +def _blank(sql: str, quote_chars: str) -> str: + """`sql` with every comment blanked, plus any quoted span opened by a character in + `quote_chars`, plus every dollar-quoted `$tag$...$tag$` span. + + Two callers need two different blindnesses: the shape check does not care what a string + literal's characters are, so it blanks both quote styles; the reserved-name check must not + let a quoted identifier hide inside a span it no longer looks at, so it blanks only string + literals and leaves double-quoted identifiers as text. Dollar-quoting is blanked for both + callers regardless of `quote_chars` — it is never an identifier, only ever a literal, and a + `'` or `;` sitting inside one is exactly what both callers must not see as SQL. + + A `'` immediately after a bare `E`/`e` opens an escape string, where a `\\` escapes whatever + follows it the same way doubling the quote does. Missing that let a `\\'` inside one close + the literal early: the real closing quote right after it then read as opening a fresh span, + and everything up to the next quote — semicolon, second statement and all — vanished into + it as though it were still part of the string. + """ + out: list[str] = [] + i, n = 0, len(sql) + while i < n: + if sql.startswith("--", i): + end = sql.find("\n", i) + i = n if end == -1 else end + 1 + continue + if sql.startswith("/*", i): + end = sql.find("*/", i + 2) + i = n if end == -1 else end + 2 + continue + char = sql[i] + if char == "$": + dollar_end = _dollar_quote_end(sql, i) + if dollar_end is not None: + i = dollar_end + out.append(" ") + continue + if char in quote_chars: + escapes = ( + char == "'" + and i > 0 + and sql[i - 1] in "Ee" + and (i == 1 or not _is_word_char(sql[i - 2])) + ) + i += 1 + while i < n: + if escapes and sql[i] == "\\": + i += 2 + continue + if sql[i] == char: + if sql[i : i + 2] == char * 2: # an escaped quote inside the literal + i += 2 + continue + i += 1 + break + i += 1 + out.append(" ") + continue + out.append(char) + i += 1 + return "".join(out) + + +def _sql_skeleton(sql: str) -> str: + """`sql` with every comment and string literal blanked out to a single space. + + The token check only needs to know what kind of statement this is and how many of them + there are; without this a semicolon or the word FOR inside a quoted value would count as + SQL, and the check would end up rejecting a value instead of a statement. + """ + return _blank(sql, "'\"") + + +def _names_the_reserved_table(sql: str) -> bool: + """Whether `_alk_conformance` appears anywhere as an identifier, quoted or not. + + Blanked for string literals only, deliberately not double-quoted identifiers — an + identifier is exactly where this name could hide from a check that blanked those away too. + The boundary either side of the name is `\\w`, not a quote: a `"` sitting right against it + is exactly the character that must not shield it, or `"_alk_conformance"` would read as + hidden the same way a comment or a literal already is. + """ + identifiers = _blank(sql, "'") + pattern = rf"(? None: + body = _sql_skeleton(sql).strip() + if not body: + raise WorldQueryRejected("query() was given nothing to run.") + unterminated = body[:-1].strip() if body.endswith(";") else body + if ";" in unterminated: + raise WorldQueryRejected("query() runs one statement; this text holds more than one.") + leading = re.match(r"[A-Za-z_]+", unterminated) + word = leading.group(0).lower() if leading else "" + if word not in _READ_KEYWORDS: + raise WorldQueryRejected( + f"query() only reads: it takes SELECT, WITH or VALUES, not {word or sql[:20]!r}." + ) + if re.search(r"\bfor\s+(update|share)\b", unterminated, re.IGNORECASE): + raise WorldQueryRejected( + "query() runs on a read-only transaction; FOR UPDATE/FOR SHARE lock rows for a " + "write that can never follow." + ) + if _names_the_reserved_table(sql): + raise WorldQueryRejected( + f"query() refuses to name {CONFORMANCE_TABLE!r}; it is the harness's own " + "conformance canary, not scenario data." + ) diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index e60b1e1a..9deb5ac9 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -83,7 +83,7 @@ def find(self, collection: str, **fields: Any) -> list[dict[str, Any]]: if all(record.get(field) == value for field, value in fields.items()) ] - def add(self, collection: str, record: Mapping[str, Any]) -> int: + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: return self.store.add(collection, record) diff --git a/src/fi/alk/harness/world/stores/__init__.py b/src/fi/alk/harness/world/stores/__init__.py index 438ace01..4f77002a 100644 --- a/src/fi/alk/harness/world/stores/__init__.py +++ b/src/fi/alk/harness/world/stores/__init__.py @@ -85,7 +85,8 @@ def collections(self) -> list[str]: ... def holds(self, collection: str) -> bool: ... def records(self, collection: str) -> list[dict[str, Any]]: ... def state(self) -> dict[str, list[dict[str, Any]]]: ... - def add(self, collection: str, record: Mapping[str, Any]) -> int: ... + def table(self, name: str) -> list[dict[str, Any]]: ... + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: ... def amend( self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" ) -> int: ... @@ -154,7 +155,13 @@ def query(self, statement: str, params: Sequence[Any] = ()) -> list[dict[str, An "records() or state()." ) - def add(self, collection: str, record: Mapping[str, Any]) -> int: + def table(self, name: str) -> list[dict[str, Any]]: + raise StoreError( + f"{self.engine} does not read one table at a time. Read what it holds with " + "records() or state()." + ) + + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: raise StoreError(_UNWRITABLE.format(engine=self.engine, verb="add to")) def amend( diff --git a/src/fi/alk/harness/world/stores/postgres.py b/src/fi/alk/harness/world/stores/postgres.py index 0a89b444..71fbb131 100644 --- a/src/fi/alk/harness/world/stores/postgres.py +++ b/src/fi/alk/harness/world/stores/postgres.py @@ -22,6 +22,7 @@ from pathlib import Path from typing import Any +from ..errors import WorldQueryRejected from . import Held, Snapshot, StoreError from .container import ContainerStore, docker @@ -93,6 +94,34 @@ def execute(self, statement: str, params: Sequence[Any] = ()) -> int: cursor = connection.execute(statement, tuple(params)) return max(0, int(cursor.rowcount or 0)) + def query(self, statement: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + """Run one read statement, on a connection Postgres itself will not let write. + + Autocommit, with the session's own default flipped to read-only rather than one shared + ``SET TRANSACTION READ ONLY`` transaction: every statement becomes its own implicit + read-only transaction, so nothing here is ever held open, and ``reset``'s drop of the + database can always proceed regardless of what a caller just read. + + An empty ``params`` tuple is passed through as ``None`` rather than as itself: psycopg + scans for placeholders whenever it is handed anything other than ``None``, and an + ordinary ``LIKE '%turkey%'`` with nothing to bind then reads its own ``%t`` as an + unmatched one and raises before the statement ever reaches Postgres. + """ + with _psycopg().connect( + self.dsn(), autocommit=True, options="-c default_transaction_read_only=on" + ) as connection: + cursor = connection.execute(statement, tuple(params) if params else None) + columns = [description[0] for description in cursor.description or []] + seen: set[str] = set() + for column in columns: + if column in seen: + raise WorldQueryRejected( + f"query() returned more than one column named {column!r}; alias one " + "of them so a row does not silently lose one under the other." + ) + seen.add(column) + return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + def _tables(self, connection: Any) -> list[str]: rows = connection.execute( "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename" @@ -100,16 +129,24 @@ def _tables(self, connection: Any) -> list[str]: return [row[0] for row in rows] def _primary_key(self, connection: Any, table: str) -> list[str]: - """The primary key columns, used only to read rows back in a stable order.""" + """The primary key columns, used only to read rows back in a stable order. + + Joined through ``pg_class``/``pg_namespace`` rather than a ``%s::regclass`` cast over an + f-string, so a table name is only ever a bound value — an embedded ``"`` (a table + created as ``CREATE TABLE "we""ird" (...)``) is just a character in that value instead + of something a regclass cast has to parse. + """ rows = connection.execute( """ SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) - WHERE i.indrelid = %s::regclass AND i.indisprimary + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = %s AND i.indisprimary ORDER BY array_position(i.indkey, a.attnum) """, - (f'public."{table}"',), + (table,), ).fetchall() return [row[0] for row in rows] @@ -125,28 +162,47 @@ def _column_types(self, connection: Any, table: str) -> dict[str, str]: ).fetchall() return {row[0]: row[1] for row in rows} - def state(self) -> dict[str, list[dict[str, Any]]]: + def _select_ordered(self, connection: Any, table: str) -> list[dict[str, Any]]: + """Every row of one table, ordered by its primary key where it has one. + + Without that order the same data comes back in whatever sequence the heap happens to + hold it, and a check comparing the first row is reading a coin toss rather than the + agent's behaviour. Built with ``sql.Identifier`` rather than an f-string because + ``table`` is a name the harness only just read out of the catalogue, not a literal it + wrote itself. + """ + sql = _psycopg().sql + key = self._primary_key(connection, table) + statement = sql.SQL("SELECT * FROM {}").format(sql.Identifier(table)) + if key: + statement += sql.SQL(" ORDER BY ") + sql.SQL(", ").join( + sql.Identifier(column) for column in key + ) + cursor = connection.execute(statement) + columns = [description[0] for description in cursor.description or []] + return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + + def state(self, only: Sequence[str] | None = None) -> dict[str, list[dict[str, Any]]]: """Every table and its rows, in the shape the checks already expect. - Ordered by primary key where there is one. Without that the same data comes back in - whatever order the heap happens to hold it, and a check comparing the first row is - reading a coin toss rather than the agent's behaviour. + ``only`` narrows the read to the named tables, still inside the one connection — a + caller that already knows it wants a subset (``HostedWorld`` excluding over-cap tables) + never pays to read and discard rows for the ones it does not. ``None`` reads every table + in ``public``, exactly as before. """ with self._connect() as connection: - out: dict[str, list[dict[str, Any]]] = {} - for table in self._tables(connection): - key = self._primary_key(connection, table) - order = ( - " ORDER BY " + ", ".join(f'"{column}"' for column in key) - if key - else "" - ) - cursor = connection.execute(f'SELECT * FROM "{table}"{order}') - columns = [description[0] for description in cursor.description or []] - out[table] = [ - dict(zip(columns, row, strict=True)) for row in cursor.fetchall() - ] - return out + tables = self._tables(connection) if only is None else list(only) + return {table: self._select_ordered(connection, table) for table in tables} + + def table(self, name: str) -> list[dict[str, Any]]: + """One table's rows, ordered by primary key where it has one. + + The read-side counterpart to ``state()``'s per-table loop, for a caller that wants only + one of them: still a single connection, so reading one table never costs a second round + trip just to learn how to order it. + """ + with self._connect() as connection: + return self._select_ordered(connection, name) # -- how to put it back ---------------------------------------------------------- @@ -244,30 +300,57 @@ def load_from(self, path: str | Path) -> None: # -- what a scenario changes ----------------------------------------------------- - def add(self, collection: str, record: Any) -> int: + def add(self, collection: str, record: Any) -> dict[str, Any]: + """Insert one record and hand back exactly what Postgres stored. + + ``RETURNING *`` rather than a second read: a caller after the row's generated key (an + identity column, a default, a trigger) would otherwise have to guess which column that + is, and a table with no natural way to re-select the row it just inserted could not be + read back at all. + """ columns = list(record) - quoted = ", ".join(f'"{column}"' for column in columns) - placeholders = ", ".join(["%s"] * len(columns)) + sql = _psycopg().sql + statement = sql.SQL("INSERT INTO {} ({}) VALUES ({}) RETURNING *").format( + sql.Identifier(collection), + sql.SQL(", ").join(sql.Identifier(column) for column in columns), + sql.SQL(", ").join(sql.SQL("%s") for _ in columns), + ) with self._connect() as connection: types = self._column_types(connection, collection) cursor = connection.execute( - f'INSERT INTO "{collection}" ({quoted}) VALUES ({placeholders})', + statement, tuple( _adapt(record[column], types.get(column, "")) for column in columns ), ) - return cursor.rowcount + stored = cursor.fetchone() + if stored is None: + raise StoreError( + f"INSERT INTO {collection!r} ... RETURNING * came back with no row; a rule " + "or a BEFORE INSERT trigger the agent's own migrations declared can turn an " + "insert into a no-op, and put() cannot report a record that was never " + "written." + ) + out = [description[0] for description in cursor.description or []] + return dict(zip(out, stored, strict=True)) def amend(self, collection: str, key: str, changes: Any, *, by: str = "") -> int: if not by: raise StoreError( f"{collection} is a table, so changing a record needs the column it is keyed on" ) - sets = ", ".join(f'"{column}" = %s' for column in changes) + sql = _psycopg().sql + statement = sql.SQL("UPDATE {} SET {} WHERE {} = %s").format( + sql.Identifier(collection), + sql.SQL(", ").join( + sql.SQL("{} = %s").format(sql.Identifier(column)) for column in changes + ), + sql.Identifier(by), + ) with self._connect() as connection: types = self._column_types(connection, collection) cursor = connection.execute( - f'UPDATE "{collection}" SET {sets} WHERE "{by}" = %s', + statement, ( *( _adapt(value, types.get(column, "")) @@ -283,9 +366,10 @@ def remove(self, collection: str, key: str = "", *, by: str = "") -> int: raise StoreError( f"{collection} is a table, so removing one record needs the column it is keyed on" ) - statement = f'DELETE FROM "{collection}"' + ( - f' WHERE "{by}" = %s' if key else "" - ) + sql = _psycopg().sql + statement = sql.SQL("DELETE FROM {}").format(sql.Identifier(collection)) + if key: + statement += sql.SQL(" WHERE {} = %s").format(sql.Identifier(by)) with self._connect() as connection: cursor = connection.execute(statement, (key,) if key else ()) return cursor.rowcount diff --git a/tests/harness/test_world_handle.py b/tests/harness/test_world_handle.py new file mode 100644 index 00000000..2c880ff8 --- /dev/null +++ b/tests/harness/test_world_handle.py @@ -0,0 +1,701 @@ +"""The hosted world handle: `HostedWorld`, its read-only view, and the postgres deltas under it. + +Two lanes, like the store tests this sits beside. Everything that is `HostedWorld`'s own +judgement — the token check, the reserved-name refusal, the row-count cap, the by-resolution, +the read-only view, `call`'s refusal — is proven against a fake store and needs neither Docker +nor a real database nor even psycopg installed. Only "is this really enforced by Postgres, not +just by us" needs the container, and those tests are skipped without one, following the same +rule as the bench Docker lane. +""" + +from __future__ import annotations + +import random +from typing import Any + +import pytest + +from fi.alk.bench._docker import docker_available +from fi.alk.harness.world.errors import ( + WorldError, + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from fi.alk.harness.world.handle import CONFORMANCE_TABLE, STATE_ROW_CAP, HostedWorld +from fi.alk.harness.world.stores import Snapshot +from fi.alk.harness.world.stores.postgres import PostgresStore + +# --- errors.py: the vocabulary, offline --------------------------------------------------- + + +@pytest.mark.parametrize( + "kind", + [ + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, + ], +) +def test_every_world_exception_is_a_world_error(kind) -> None: + assert issubclass(kind, WorldError) + assert isinstance(kind("because"), RuntimeError) + + +# --- a fake store: HostedWorld's own judgement, without a database -------------------------- + + +class FakeStore: + """Just enough of `PostgresStore` for `HostedWorld` to run on, without a database. + + `table`/`state` mirror the store's own per-table ordering directly off `primary_keys`, the + way `PostgresStore` resolves it itself; `query` only fakes the three catalogue lookups + `HostedWorld` actually issues (`pg_tables`, `pg_index`, `information_schema.columns`) plus + one quoted-table-name passthrough for `world.query()` itself, since nothing here needs to + parse real SQL to prove what is `HostedWorld`'s own judgement rather than the database's. + """ + + def __init__( + self, + tables: dict[str, list[dict[str, Any]]], + primary_keys: dict[str, list[str]] | None = None, + columns: dict[str, set[str]] | None = None, + ) -> None: + self.tables = {name: list(rows) for name, rows in tables.items()} + self.primary_keys = primary_keys or {} + # A table's columns otherwise come from whatever its rows happen to hold, which is + # nothing for an empty table; an explicit map lets a test give a table its columns + # without needing a seed row just to make a by= resolve, the way it would against a + # real, empty Postgres table. + self.columns = columns or {} + + def _ordered(self, name: str) -> list[dict[str, Any]]: + rows = self.tables.get(name, []) + key = self.primary_keys.get(name) + if key: + return sorted((dict(row) for row in rows), key=lambda row: row[key[0]]) + return [dict(row) for row in rows] + + def state(self, only: list[str] | None = None) -> dict[str, list[dict[str, Any]]]: + names = self.tables if only is None else only + return {name: self._ordered(name) for name in names} + + def table(self, name: str) -> list[dict[str, Any]]: + return self._ordered(name) + + def query(self, statement: str, params: tuple = ()) -> list[dict[str, Any]]: + if "pg_tables" in statement: + return [{"tablename": name} for name in sorted(self.tables)] + if "pg_index" in statement: + table = params[0] + return [{"attname": column} for column in self.primary_keys.get(table, [])] + if "information_schema.columns" in statement: + table = params[0] + known = self.columns.get(table) + if known is None: + known = {column for row in self.tables.get(table, []) for column in row} + return [{"column_name": column} for column in sorted(known)] + for name, rows in self.tables.items(): + if f'"{name}"' in statement: + return [dict(row) for row in rows] + return [] + + def add(self, collection: str, record: dict[str, Any]) -> dict[str, Any]: + stored = dict(record) + stored.setdefault("id", len(self.tables[collection]) + 1) + self.tables[collection].append(stored) + return stored + + def amend(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + changed = 0 + for row in self.tables.get(collection, []): + if str(row.get(by)) == str(key): + row.update(changes) + changed += 1 + return changed + + def remove(self, collection: str, key: str = "", *, by: str = "") -> int: + rows = self.tables.get(collection, []) + if not key: + count = len(rows) + rows.clear() + return count + kept = [row for row in rows if str(row.get(by)) != str(key)] + removed = len(rows) - len(kept) + rows[:] = kept + return removed + + +def _world( + tables: dict[str, list[dict[str, Any]]], + *, + baseline: dict[str, int] | None = None, + primary_keys: dict[str, list[str]] | None = None, + columns: dict[str, set[str]] | None = None, +) -> HostedWorld: + visible = {name: len(rows) for name, rows in tables.items() if name != CONFORMANCE_TABLE} + store = FakeStore(tables, primary_keys, columns) + return HostedWorld( + store, + world_index=3, + rng=random.Random(7), + baseline_row_counts=baseline if baseline is not None else visible, + ) + + +def test_world_index_and_rng_are_carried_through_unchanged() -> None: + rng = random.Random(11) + world = HostedWorld( + FakeStore({"orders": []}), world_index=4, rng=rng, baseline_row_counts={"orders": 0} + ) + assert world.world_index == 4 + assert world.rng is rng + + +# --- state() ----------------------------------------------------------------------------- + + +def test_state_reports_every_visible_table_empty_ones_included() -> None: + world = _world({"customers": [{"id": 1, "name": "ana"}], "orders": []}) + assert world.state() == {"customers": [{"id": 1, "name": "ana"}], "orders": []} + + +def test_state_excludes_the_conformance_table() -> None: + world = _world( + {"orders": [{"id": 1}], CONFORMANCE_TABLE: [{"id": 1, "marker": "alive"}]}, + ) + assert CONFORMANCE_TABLE not in world.state() + + +def test_state_selects_one_table() -> None: + world = _world({"customers": [{"id": 1}], "orders": [{"id": 9}]}) + assert world.state("orders") == {"orders": [{"id": 9}]} + + +def test_state_selector_reflects_the_fakestores_own_ordering() -> None: + """`table()`'s primary-key ordering is `PostgresStore._select_ordered`'s job, proven against + a real database in the docker lane below; this only pins that `HostedWorld` passes through + whatever order `FakeStore.table()` hands back, not that the ordering itself is correct.""" + world = _world( + {"orders": [{"id": 2, "item": "b"}, {"id": 1, "item": "a"}]}, + primary_keys={"orders": ["id"]}, + ) + assert [row["id"] for row in world.state("orders")["orders"]] == [1, 2] + + +def test_state_selector_passes_through_the_fakes_row_order_when_it_has_no_primary_key() -> None: + """Same caveat as above: this is `FakeStore`'s own behaviour when it has no key to sort by, + not a guarantee `HostedWorld` makes about row order.""" + given = [{"item": "b"}, {"item": "a"}] + world = _world({"orders": given}) + assert world.state("orders")["orders"] == given + + +def test_naming_the_conformance_table_is_reserved_not_a_usage_mistake() -> None: + world = _world({"orders": [], CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.state(CONFORMANCE_TABLE) + + +def test_naming_a_table_this_world_does_not_have_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.state("bookings") + + +def test_zero_visible_tables_is_unavailable_not_an_empty_snapshot() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldUnavailable): + world.state() + + +def test_state_selector_on_an_empty_schema_is_unavailable_not_a_usage_error() -> None: + """The empty-schema case is `WorldUnavailable` before the membership test even runs: a world + with nothing in it cannot serve any table name, so blaming the scenario for asking about one + in particular is backwards.""" + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldUnavailable): + world.state("bookings") + + +def test_a_table_over_its_baseline_cap_raises_without_touching_its_rows() -> None: + """The cap is decided from what was measured at freeze, never recounted here. + + The fake table itself holds nothing near 5,000 rows; only the baseline claims it does. If + `state()` re-measured, this would pass instead of raising. + """ + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP + 1}) + with pytest.raises(WorldStateTooLarge): + world.state("orders") + + +def test_a_table_at_or_under_the_cap_is_read_normally() -> None: + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP}) + assert world.state("orders") == {"orders": [{"id": 1}]} + + +def test_bare_state_excludes_an_over_cap_table_without_raising() -> None: + """The exclusion is a property of the bare snapshot, not a failure — one seeded audit table + must not make the primary read verb inert for the whole run. Naming it explicitly still + raises: the exclusion never becomes a way to read the table around its own cap.""" + world = _world( + {"orders": [{"id": 1}], "audit_log": [{"id": 1}]}, + baseline={"orders": 1, "audit_log": STATE_ROW_CAP + 1}, + ) + snapshot = world.state() + assert snapshot == {"orders": [{"id": 1}]} + assert "audit_log" not in snapshot + with pytest.raises(WorldStateTooLarge): + world.state("audit_log") + + +def test_bare_state_raises_when_every_table_is_over_the_cap() -> None: + """Excluding every over-cap table would leave the snapshot `{}` while the schema is not + empty — exactly the vacuous observation `state()` must never produce, since a check reading + an empty snapshot as "nothing there" would pass on a world it never actually looked at — so + the bare call raises instead of quietly handing that back.""" + world = _world( + {"orders": [{"id": 1}], "audit_log": [{"id": 1}]}, + baseline={"orders": STATE_ROW_CAP + 1, "audit_log": STATE_ROW_CAP + 1}, + ) + with pytest.raises(WorldStateTooLarge): + world.state() + + +def test_a_table_missing_from_the_baseline_dict_is_unavailable_not_under_cap() -> None: + """A missing entry used to default to a row count of 0 and read as under the cap; the cap + cannot be decided for a table nobody measured at freeze. The check runs once, at + construction, rather than waiting for a scenario's first access to discover the gap.""" + with pytest.raises(WorldUnavailable): + _world({"orders": [{"id": 1}]}, baseline={}) + + +# --- put / change / drop ------------------------------------------------------------------- + + +def test_put_returns_the_stored_record_key_included() -> None: + world = _world({"customers": []}) + stored = world.put("customers", {"name": "ana"}) + assert stored == {"id": 1, "name": "ana"} + + +def test_put_into_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: []}) + with pytest.raises(WorldReservedName): + world.put(CONFORMANCE_TABLE, {"marker": "x"}) + + +def test_put_into_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"customers": []}) + with pytest.raises(WorldUsageError): + world.put("not_a_table", {"name": "ana"}) + + +def test_put_with_a_key_is_a_usage_error() -> None: + """A hosted table's key is the table's own; passing one is telling it what to call a value + the table is about to generate itself.""" + world = _world({"customers": []}) + with pytest.raises(WorldUsageError): + world.put("customers", {"name": "ana"}, key="c1") + + +def test_change_without_by_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.change("orders", "1", {"item": "b"}) + + +def test_change_with_by_updates_and_reports_the_count() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + assert world.change("orders", "1", {"item": "b"}, by="id") == 1 + + +def test_change_without_by_resolves_a_single_column_primary_key() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}, primary_keys={"orders": ["id"]}) + assert world.change("orders", "1", {"item": "b"}) == 1 + + +def test_change_on_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.change("not_a_table", "1", {"item": "b"}, by="id") + + +def test_change_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.change(CONFORMANCE_TABLE, "1", {"marker": "x"}, by="id") + + +def test_change_with_a_by_that_is_not_a_column_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.change("orders", "1", {"item": "b"}, by="nope") + + +def test_change_by_a_column_works_on_an_empty_table_when_the_fake_declares_its_columns() -> None: + """Proves the columns map on `FakeStore` actually does something: without it an empty + table's columns default to empty and this would raise `WorldUsageError` in the fake even + though a real, empty Postgres table would resolve `by="id"` just fine.""" + world = _world({"orders": []}, columns={"orders": {"id", "item"}}) + assert world.change("orders", "1", {"item": "b"}, by="id") == 0 + + +def test_drop_with_a_key_and_no_by_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldUsageError): + world.drop("orders", "1") + + +def test_drop_with_no_key_needs_no_by() -> None: + world = _world({"orders": [{"id": 1}, {"id": 2}]}) + assert world.drop("orders") == 2 + + +def test_drop_without_by_resolves_a_single_column_primary_key() -> None: + world = _world({"orders": [{"id": 1}, {"id": 2}]}, primary_keys={"orders": ["id"]}) + assert world.drop("orders", "1") == 1 + + +def test_drop_on_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.drop("not_a_table", "1", by="id") + + +def test_drop_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.drop(CONFORMANCE_TABLE) + + +def test_drop_with_a_by_that_is_not_a_column_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.drop("orders", "1", by="nope") + + +# --- query(): the token check ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "sql", + [ + "SELECT * FROM orders", + "select * from orders", + " -- a leading comment\nSELECT * FROM orders", + "/* block */ SELECT * FROM orders", + "WITH recent AS (SELECT * FROM orders) SELECT * FROM recent", + "VALUES (1), (2)", + "SELECT * FROM orders;", + "SELECT * FROM orders WHERE note = 'ends with a semicolon;'", + "SELECT * FROM orders WHERE note = 'ask for update'", + "SELECT * FROM orders WHERE note = E'a semicolon inside an escape string: ;'", + "SELECT * FROM orders WHERE note = E'it\\'s for update'", + ], +) +def test_query_accepts_one_read_statement(sql) -> None: + """Not rejected, whatever it finds to read - the token check is a shape question only.""" + world = _world({"orders": [{"id": 1}]}) + assert isinstance(world.query(sql), list) + + +@pytest.mark.parametrize( + "sql", + [ + "UPDATE orders SET item = 'x'", + "DELETE FROM orders", + "INSERT INTO orders (item) VALUES ('x')", + "SELECT * FROM orders; DROP TABLE orders", + "SELECT * FROM orders FOR UPDATE", + "SELECT * FROM orders FOR SHARE", + " ", + ], +) +def test_query_rejects_anything_that_is_not_one_plain_read(sql) -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query(sql) + + +def test_query_accepts_a_semicolon_inside_a_dollar_quoted_literal() -> None: + """A dollar-quoted span is a literal like any other quoted one; a `;` inside `$$...$$` must + not count toward the one-statement rule any more than one inside `'...'` does.""" + world = _world({"orders": [{"id": 1}]}) + assert isinstance(world.query("SELECT $$contains a semicolon: ;$$ AS note"), list) + + +def test_query_rejects_a_second_statement_hidden_behind_a_dollar_quoted_apostrophe() -> None: + """Before `_blank` knew dollar-quoting, the `'` inside `$$it's fine$$` opened a bogus quote + span that swallowed everything after it, including the real statement separator, so a + second statement rode through unrejected. Recognising the dollar-quoted span as one unit is + what keeps the semicolon between the two real statements visible.""" + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query("SELECT $$it's fine$$ AS a; SELECT 2") + + +def test_query_rejects_a_second_statement_hidden_behind_an_escaped_quote() -> None: + """Before `_blank` knew `E'...'`'s backslash escapes, the `\\'` inside one closed the + literal early; the real closing quote right after it then read as opening a fresh span, and + everything up to the next quote — semicolon, second statement and all — vanished into it + unrejected.""" + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query("SELECT * FROM orders WHERE note = E'x\\'y' ; DELETE FROM orders") + + +@pytest.mark.parametrize( + "sql", + [ + f"SELECT * FROM {CONFORMANCE_TABLE}", + f'SELECT * FROM "{CONFORMANCE_TABLE}"', + ], +) +def test_query_rejects_naming_the_conformance_table(sql) -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query(sql) + + +# --- read_only(): the view ready() and check() actually get --------------------------------- + + +def test_read_only_carries_world_index_and_rng() -> None: + rng = random.Random(5) + world = HostedWorld( + FakeStore({"orders": []}), world_index=2, rng=rng, baseline_row_counts={"orders": 0} + ) + view = world.read_only() + assert view.world_index == 2 + assert view.rng is rng + + +def test_read_only_still_reads() -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + assert view.state() == {"orders": [{"id": 1}]} + assert view.query('SELECT * FROM "orders"') == [{"id": 1}] + + +@pytest.mark.parametrize( + "act", + [ + lambda view: view.put("orders", {"item": "x"}), + lambda view: view.change("orders", "1", {"item": "x"}, by="id"), + lambda view: view.drop("orders", "1", by="id"), + lambda view: view.call("some_tool"), + ], +) +def test_read_only_refuses_every_write_verb(act) -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + with pytest.raises(WorldReadOnly): + act(view) + + +def test_read_only_hides_unknown_capability_probes_behind_a_plain_attribute_error() -> None: + """`prove.py`/`probe.py`/`run/voice.py` all reach for capability attributes on world objects + through `hasattr`/`getattr(..., default)`; only a plain `AttributeError` makes that pattern + work, so an unknown name here must read as one instead of as a write refusal.""" + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + assert hasattr(view, "forward") is False + assert getattr(view, "runtime_tools", set()) == set() + + +def test_read_only_dunder_lookups_stay_plain_attribute_errors() -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + with pytest.raises(AttributeError): + view.__wrapped__ + + +# --- call(): not implemented, the http_tool shim's wire format is unpinned ------------------ + + +def test_call_is_unavailable() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUnavailable): + world.call("some_tool") + + +# --- with docker: the deltas that only Postgres itself can really prove -------------------- + +pg = pytest.mark.skipif(not docker_available(), reason="docker daemon unavailable") + +SCHEMA = """ +CREATE TABLE _alk_conformance ( + id serial PRIMARY KEY, + marker text NOT NULL +); +CREATE TABLE customers ( + id serial PRIMARY KEY, + name text NOT NULL +); +CREATE TABLE orders ( + id serial PRIMARY KEY, + customer_id int NOT NULL REFERENCES customers(id), + item text NOT NULL +); +""" + +SEED = """ +INSERT INTO _alk_conformance (marker) VALUES ('alive'); +INSERT INTO customers (name) VALUES ('ana'), ('bo'); +INSERT INTO orders (customer_id, item) VALUES (1, 'turkey'); +""" + + +@pytest.fixture(scope="module") +def store(): + running = PostgresStore(version="16") + running.start() + try: + running.apply(SCHEMA) + yield running + finally: + running.stop() + + +@pytest.fixture() +def seeded(store): + store.restore(Snapshot()) + store.apply(SEED) + return store + + +@pg +def test_postgres_query_reads_real_rows(seeded) -> None: + rows = seeded.query('SELECT * FROM "customers" ORDER BY "id"') + assert [row["name"] for row in rows] == ["ana", "bo"] + + +@pg +def test_postgres_query_refuses_a_write_at_the_database(seeded) -> None: + """The friendliness check lives in `HostedWorld`; the store's own guard is this.""" + import psycopg + + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + seeded.query("DELETE FROM orders") + # And the delete really did not happen. + assert seeded.state()["orders"] + + +@pg +def test_postgres_query_accepts_a_percent_literal_with_no_bound_params(seeded) -> None: + """An empty params tuple is still `not None` to psycopg's placeholder scanner, which would + otherwise read the `%t` in `%turkey%` as an unmatched placeholder and raise before the + statement ever reaches Postgres.""" + rows = seeded.query("SELECT * FROM orders WHERE item LIKE '%turkey%'") + assert [row["item"] for row in rows] == ["turkey"] + + +@pg +def test_postgres_add_returns_the_row_with_its_generated_key(seeded) -> None: + stored = seeded.add("customers", {"name": "cy"}) + assert stored["name"] == "cy" + assert isinstance(stored["id"], int) + assert stored in seeded.state()["customers"] + + +@pg +def test_hosted_world_query_refuses_a_data_modifying_cte(seeded) -> None: + """A data-modifying CTE passes the token check (its leading keyword is WITH, a read + keyword); the read-only transaction underneath is what actually stops the INSERT it + hides, exactly as the friendliness check was always meant to be backed up.""" + import psycopg + + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(9), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + world.query( + "WITH w AS (INSERT INTO orders (customer_id, item) VALUES (1, 'ham') " + "RETURNING *) SELECT * FROM w" + ) + assert len(world.state()["orders"]) == 1 + + +@pg +def test_hosted_world_query_refuses_a_result_with_duplicate_column_labels(seeded) -> None: + """`o.*, c.*` on a join gives both tables' `id` the same label; silently keeping only the + last one under `dict(zip(...))` would let a check read the wrong table's key with no error + anywhere - refusing beats a check that is quietly wrong.""" + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(3), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + with pytest.raises(WorldQueryRejected): + world.query("SELECT o.*, c.* FROM orders o JOIN customers c ON c.id = o.customer_id") + + +@pg +def test_hosted_world_end_to_end_against_a_real_database(seeded) -> None: + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(42), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + + state = world.state() + assert CONFORMANCE_TABLE not in state + assert [row["name"] for row in state["customers"]] == ["ana", "bo"] + + with pytest.raises(WorldReservedName): + world.state(CONFORMANCE_TABLE) + + stored = world.put("customers", {"name": "cy"}) + assert stored["name"] == "cy" + + changed = world.change("orders", "1", {"item": "ham"}, by="id") + assert changed == 1 + assert world.state("orders")["orders"][0]["item"] == "ham" + + dropped = world.drop("orders", "1", by="id") + assert dropped == 1 + assert world.state("orders") == {"orders": []} + + assert world.query("SELECT name FROM customers ORDER BY name") == [ + {"name": "ana"}, + {"name": "bo"}, + {"name": "cy"}, + ] + + +@pg +def test_hosted_world_cap_uses_the_baseline_not_a_live_count(seeded) -> None: + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(1), + baseline_row_counts={"customers": STATE_ROW_CAP + 1, "orders": 1}, + ) + with pytest.raises(WorldStateTooLarge): + world.state("customers") + # A table the baseline did not flag reads normally in the same call. + assert world.state("orders") == {"orders": [{"id": 1, "customer_id": 1, "item": "turkey"}]} + + +@pg +def test_hosted_world_on_an_empty_schema_is_unavailable() -> None: + empty = PostgresStore(version="16") + empty.start() + try: + world = HostedWorld(empty, world_index=0, rng=random.Random(1), baseline_row_counts={}) + with pytest.raises(WorldUnavailable): + world.state() + finally: + empty.stop() From f286df7b47066854f277c059c7139b10d3857b30 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 01:46:13 +0530 Subject: [PATCH 02/20] feat(harness): align runner return conventions with the world-handle contract check(): empty/whitespace strings count as held; any non-string, non-False value is a broken check, not an agent failure. ready(): bare False and any non-string value are broken ready code; setup stays advisory. Bare state() excludes unmeasured tables the same way it excludes over-cap ones, so nothing the agent does during a call can change which tables raise; the typed refusal stays on the explicit selector. Signed-off-by: khushalsonawat --- src/fi/alk/harness/checks.py | 14 +++- src/fi/alk/harness/folder.py | 13 ++++ src/fi/alk/harness/world/handle.py | 42 +++++++++-- tests/harness/test_runner_conventions.py | 96 ++++++++++++++++++++++++ tests/harness/test_world_handle.py | 25 ++++++ tests/test_harness.py | 5 +- 6 files changed, 184 insertions(+), 11 deletions(-) create mode 100644 tests/harness/test_runner_conventions.py diff --git a/src/fi/alk/harness/checks.py b/src/fi/alk/harness/checks.py index f17d2e41..e5dc01ad 100644 --- a/src/fi/alk/harness/checks.py +++ b/src/fi/alk/harness/checks.py @@ -73,9 +73,19 @@ def run_check( broken=True, ) - if said is None or said is True: + if said is None or said is True or (isinstance(said, str) and not said.strip()): return Outcome(name, True) - return Outcome(name, False, str(said) if said is not True else "") + if said is False: + return Outcome(name, False, "False") + if not isinstance(said, str): + return Outcome( + name, + False, + f"the check returned {type(said).__name__} {said!r}; a check returns a sentence " + "naming what is wrong, or None when it held.", + broken=True, + ) + return Outcome(name, False, said) def all_held(outcomes: Sequence[Outcome]) -> bool: diff --git a/src/fi/alk/harness/folder.py b/src/fi/alk/harness/folder.py index 2b5838ad..d9864126 100644 --- a/src/fi/alk/harness/folder.py +++ b/src/fi/alk/harness/folder.py @@ -99,10 +99,23 @@ def _run(source: str, name: str, entry: str, *args: Any) -> Outcome: if said is None or said is True or (isinstance(said, str) and not said.strip()): return Outcome(True) if said is False: + # Bare False from ready() names no precondition, so a scenario that hits it cannot be + # told apart from one whose ready.py is simply wrong — that is our mistake, not a + # generation-time precondition failure, so ready() alone reports it broken. return Outcome( False, f"{name} returned False without saying what is wrong. Return the sentence instead, " "or None if it holds.", + broken=(entry == "ready"), + ) + if entry == "ready" and not isinstance(said, str): + # Same reasoning as bare False, widened: ready() has no way to turn a non-string value + # into a precondition sentence, so any of them is our mistake rather than the world's. + return Outcome( + False, + f"{name} returned {type(said).__name__} {said!r}. Return the sentence naming what is " + "missing, or None if it holds.", + broken=True, ) return Outcome(False, str(said)) diff --git a/src/fi/alk/harness/world/handle.py b/src/fi/alk/harness/world/handle.py index 58481bee..b5e2699c 100644 --- a/src/fi/alk/harness/world/handle.py +++ b/src/fi/alk/harness/world/handle.py @@ -85,9 +85,12 @@ def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: Bare `state()` leaves an over-cap table out of the snapshot rather than raising through it — one seeded audit table must not make the primary read verb inert for the whole - run. Naming that table explicitly (`state("big_table")`) still raises: the exclusion is - a property of the snapshot, not a way to read the table around its own cap. If every - visible table is over cap the exclusion would leave the snapshot `{}` — the one thing + run. A table nothing measured at baseline freeze gets the same treatment: the only way + one exists is a table the agent under test created since construction, and nothing it + does during a call may decide whether bare `state()` raises. Naming either kind of table + explicitly (`state("big_table")`) still raises: the exclusion is a property of the + snapshot, not a way to read the table around its own cap. If every visible table is + over-cap or unmeasured, the exclusion would leave the snapshot `{}` — the one thing state() must never return, since an empty snapshot reads as an observation and makes a negative check pass on a world nobody actually looked at — so that case raises too, naming the tables it would have excluded. The exclusion itself happens at the read: the @@ -103,9 +106,10 @@ def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: raise WorldUsageError( f"{table!r} is not a table in this world; it holds {sorted(visible)}." ) - if self._baseline_row_counts[table] > STATE_ROW_CAP: + count = self._row_count(table) + if count > STATE_ROW_CAP: raise WorldStateTooLarge( - f"{table!r} held {self._baseline_row_counts[table]} rows when the baseline " + f"{table!r} held {count} rows when the baseline " f"was frozen, over the {STATE_ROW_CAP}-row cap; state() will not read it " "back." ) @@ -113,11 +117,21 @@ def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: names = self._visible_tables() self._require_nonempty_schema(names) - included = [name for name in names if self._baseline_row_counts[name] <= STATE_ROW_CAP] + # A table nothing measured at freeze is treated the same as an over-cap one here, not + # routed through `_row_count`'s typed refusal — that refusal is for a scenario naming a + # table explicitly; the bare snapshot must not go unavailable over a table the agent + # itself created since construction (the only actor besides the provisioner that can). + included = [ + name + for name in names + if name in self._baseline_row_counts + and self._baseline_row_counts[name] <= STATE_ROW_CAP + ] if not included: raise WorldStateTooLarge( f"every table this world holds — {sorted(names)} — is over the " - f"{STATE_ROW_CAP}-row cap; state() will not return {{}} in their place." + f"{STATE_ROW_CAP}-row cap or was never measured at baseline freeze; state() " + "will not return {} in their place." ) # One connection, asked for only the included tables — a bare state() used to open a # fresh connection per table (and a second one just to look up its primary key) to read @@ -246,6 +260,20 @@ def _require_baseline_coverage(self, names: list[str]) -> None: "measured at freeze." ) + def _row_count(self, table: str) -> int: + """This table's baseline row count, or the same typed refusal `_require_baseline_coverage` + gives a construction-time gap — for a table that only shows up afterward, so it never + reaches the caller as a bare `KeyError`. + """ + try: + return self._baseline_row_counts[table] + except KeyError: + raise WorldUnavailable( + f"the baseline row counts this world was built with never measured " + f"{table!r}; state()'s cap cannot be decided for a table nobody measured at " + "freeze." + ) from None + def _visible_tables(self) -> list[str]: rows = self._store.query( "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename" diff --git a/tests/harness/test_runner_conventions.py b/tests/harness/test_runner_conventions.py new file mode 100644 index 00000000..ee9984de --- /dev/null +++ b/tests/harness/test_runner_conventions.py @@ -0,0 +1,96 @@ +"""Pins the runner return conventions in world-handle-interface.md v3.2's "Return conventions" +block: a check returning ``""``/whitespace counts as held, and a bare ``False`` from ``ready()`` +is broken rather than a plain not-ready. The unchanged neighbors are pinned alongside them so a +future edit to either runner cannot drift the other conventions without a test noticing. +""" + +from __future__ import annotations + +import pytest + +from fi.alk.harness.checks import run_check +from fi.alk.harness.folder import _run, check_ready +from fi.alk.harness.scenario import Scenario + +# --- checks.py: run_check's return convention ------------------------------------------------- + + +@pytest.mark.parametrize("returning", ["None", "True", "''", "' '"]) +def test_check_returning_nothing_or_whitespace_holds(returning: str) -> None: + outcome = run_check(f"def check(world, calls):\n return {returning}\n", None, []) + assert outcome.held and not outcome.broken and outcome.said == "" + + +def test_check_returning_a_non_empty_string_does_not_hold() -> None: + outcome = run_check("def check(world, calls):\n return 'no rows'\n", None, []) + assert not outcome.held and not outcome.broken and outcome.said == "no rows" + + +def test_check_returning_bare_false_does_not_hold_with_reason_false() -> None: + """An agent result, matching checks.py's existing convention — not a broken check.""" + outcome = run_check("def check(world, calls):\n return False\n", None, []) + assert not outcome.held and not outcome.broken and outcome.said == "False" + + +@pytest.mark.parametrize("returning", ["0", "1", "42", "[]", "{}", "0.0", "object()"]) +def test_check_returning_any_other_value_is_broken(returning: str) -> None: + """Anything that is not the held set, a non-empty string, or bare False is our own mistake, + not a finding about the agent — a check that returns 0 by accident must not be scored as a + failing sub-goal with an uninterpretable reason.""" + outcome = run_check(f"def check(world, calls):\n return {returning}\n", None, []) + assert not outcome.held and outcome.broken + + +# --- folder.py: ready()'s return convention --------------------------------------------------- + + +@pytest.mark.parametrize("returning", ["None", "True", "''", "' '"]) +def test_ready_returning_nothing_or_whitespace_is_ready(returning: str) -> None: + outcome = _run(f"def ready(world):\n return {returning}\n", "s/ready.py", "ready", None) + assert outcome.ok and not outcome.broken and outcome.said == "" + + +def test_ready_returning_a_non_empty_string_is_not_ready() -> None: + outcome = _run( + "def ready(world):\n return 'orders pending'\n", "s/ready.py", "ready", None + ) + assert not outcome.ok and not outcome.broken and outcome.said == "orders pending" + + +def test_ready_returning_bare_false_is_broken_not_advisory() -> None: + """The behavioral change: ready() cannot name what it wants, so a bare False is our own + mistake, not a precondition failing on the shared sealed baseline.""" + outcome = _run("def ready(world):\n return False\n", "s/ready.py", "ready", None) + assert not outcome.ok and outcome.broken + + +def test_setup_returning_bare_false_stays_advisory_not_broken() -> None: + """The change is scoped to ready() only; setup()'s own convention is untouched.""" + outcome = _run("def setup(world):\n return False\n", "s/setup.py", "setup", None) + assert not outcome.ok and not outcome.broken + + +@pytest.mark.parametrize("returning", ["0", "1", "[]", "{}", "0.0"]) +def test_ready_returning_any_other_value_is_broken(returning: str) -> None: + """ready() cannot turn a non-string, non-False value into a precondition sentence either, so + it gets the same broken verdict bare False does.""" + outcome = _run(f"def ready(world):\n return {returning}\n", "s/ready.py", "ready", None) + assert not outcome.ok and outcome.broken + + +@pytest.mark.parametrize("returning", ["0", "1", "[]", "{}", "0.0"]) +def test_setup_returning_any_other_value_stays_advisory(returning: str) -> None: + """The widened rule is keyed on entry == "ready", not on the value alone — setup() keeps its + own untouched convention for every one of these values too.""" + outcome = _run(f"def setup(world):\n return {returning}\n", "s/setup.py", "setup", None) + assert not outcome.broken + + +def test_check_ready_reports_bare_false_as_broken() -> None: + """Pins the production entry point, not just `_run`: `check_ready` is the only caller that + can ever produce entry == "ready", so a test that only calls `_run` directly would not catch + a rename or rewiring that silently reverted this.""" + outcome = check_ready( + Scenario(name="s", ready_code="def ready(world):\n return False\n"), None + ) + assert not outcome.ok and outcome.broken diff --git a/tests/harness/test_world_handle.py b/tests/harness/test_world_handle.py index 2c880ff8..bba11686 100644 --- a/tests/harness/test_world_handle.py +++ b/tests/harness/test_world_handle.py @@ -276,6 +276,31 @@ def test_a_table_missing_from_the_baseline_dict_is_unavailable_not_under_cap() - _world({"orders": [{"id": 1}]}, baseline={}) +def test_a_table_appearing_after_construction_is_unavailable_on_the_selector_path() -> None: + """`_require_baseline_coverage` only sees the tables visible when the handle was built; a + table that shows up afterward is still absent from `_baseline_row_counts`. Naming it + explicitly must reach the typed `WorldUnavailable` naming the table, not a bare `KeyError` + off an unguarded index — a `WorldUnavailable("")` would pass this test for the wrong reason, + so the message is asserted, not just the type.""" + world = _world({"orders": [{"id": 1}]}) + world._store.tables["late_table"] = [{"id": 1}] + with pytest.raises(WorldUnavailable) as raised: + world.state("late_table") + assert "late_table" in str(raised.value) + + +def test_a_table_appearing_after_construction_is_excluded_from_the_bare_snapshot() -> None: + """Nothing the agent under test does during a call may decide whether bare `state()` raises + — a table it creates since construction is the only way an unmeasured table can exist, so the + bare path excludes it the same way it excludes an over-cap table rather than going + unavailable for the whole snapshot.""" + world = _world({"orders": [{"id": 1}]}) + world._store.tables["late_table"] = [{"id": 1}] + snapshot = world.state() + assert snapshot == {"orders": [{"id": 1}]} + assert "late_table" not in snapshot + + # --- put / change / drop ------------------------------------------------------------------- diff --git a/tests/test_harness.py b/tests/test_harness.py index 02f646e1..ffa4f364 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -5344,9 +5344,10 @@ def test_a_setup_or_ready_that_says_nothing_is_not_a_complaint(): ) assert not complained.ok and complained.said == "no pending orders" - # And False is a failure that says nothing, so the message says that rather than being blank. + # And False from ready() names no precondition, so it is broken rather than a plain failure — + # the message says that rather than being blank. bare = _run("def ready(world):\n return False\n", "s/ready.py", "ready", None) - assert not bare.ok and "without saying what is wrong" in bare.said + assert not bare.ok and bare.broken and "without saying what is wrong" in bare.said def test_an_optional_field_may_be_null(): From e656354834785c0374cb109859e7ef0832919dcd Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 02:51:25 +0530 Subject: [PATCH 03/20] feat(harness): environment-bundle.v2 manifest models Typed models for the hosted bundle manifest: runtime kinds with the evidence seam, managed/source processes, seed stores with per-protocol sentinels, capability wiring, and the byte-exact inputs-digest helper. Strategy-engine pairing keys on the capability protocol; the manifest re-gains the resolved-secret sweep, scoped so v2's own secret_purposes field is not mistaken for a credential. Riders: runnable-check predicate parity, broken-value message truncation, cap-exclusion coverage. Signed-off-by: khushalsonawat --- src/fi/alk/harness/bundle_v2.py | 535 +++++++++++++++++++++++ src/fi/alk/harness/checks.py | 6 +- src/fi/alk/harness/folder.py | 9 +- tests/harness/test_bundle_v2.py | 439 +++++++++++++++++++ tests/harness/test_runner_conventions.py | 2 +- tests/harness/test_world_handle.py | 13 + 6 files changed, 996 insertions(+), 8 deletions(-) create mode 100644 src/fi/alk/harness/bundle_v2.py create mode 100644 tests/harness/test_bundle_v2.py diff --git a/src/fi/alk/harness/bundle_v2.py b/src/fi/alk/harness/bundle_v2.py new file mode 100644 index 00000000..f5e77d3d --- /dev/null +++ b/src/fi/alk/harness/bundle_v2.py @@ -0,0 +1,535 @@ +"""`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape. + +v1 (`bundle.py`) describes a `command`-per-service compose world and embeds the repository +source. v2 describes `/work/source` as already present and a job that starts plain processes on +localhost: `processes` (managed engines and copied-and-built source trees), `seed` (how each +store's baseline is built and proven), and the same `capabilities`/`readiness`/`files`/ +`provenance` shape widened for both. v1 stays untouched — this module is additive, not a +replacement, and the two schema versions are never interchangeable: a hosted provisioner that +receives a `…bundle.v1` manifest rejects it rather than guessing. + +What lives here is model-layer only: the shapes, the closed vocabularies, and the rules that need +nothing but the manifest's own fields to decide. Rules that need the bundle's actual files (secret +scanning, digest/file verification) or the job it will run under (`compose_not_hosted`, +`engine_unsupported`, `no_sql_store`, the `depends_on` graph, placeholder-vocabulary checking, +reserved-name scanning of migration content) are the §2e preflight checklist's job, not this +module's — see `hosted-execution-seams.md` §2e. Also deferred to that preflight: translating +pydantic's `extra="forbid"` rejection of an unknown process-entry key into §2b's `unknown_field` +code — that translation belongs where error surfacing is owned, not here. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from enum import Enum +from pathlib import Path +from typing import Annotated, Literal, Sequence, Union + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator + +from .bundle import CapabilityProtocol, _reject_secret_values, _safe_relative + +BUNDLE_V2_SCHEMA_VERSION = "futureagi.environment-bundle.v2" +BUNDLE_V2_MANIFEST = "manifest.json" + + +class BundleV2Error(RuntimeError): + """A v2 bundle manifest is wrong-versioned, malformed, or fails a model-layer rule.""" + + +# --- §2a runtime ------------------------------------------------------------------------------- + + +class RuntimeKindV2(str, Enum): + PROCESS = "process" + EXTERNAL = "external" + COMPOSE = "compose" + + +class EvidenceSeam(str, Enum): + HTTP_TOOL = "http_tool" + TOOL_TRACE = "tool_trace" + + +class BundleRuntimeV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + kind: RuntimeKindV2 + control_service: str | None = None + evidence_seam: EvidenceSeam | None = None + # Carried over from v1 for `kind: compose` only (local SDK runs); a hosted `process` bundle + # has no document to point at, since `/work/source` is already on disk. + document: str | None = None + + @model_validator(mode="after") + def _kind_specific_rules(self) -> "BundleRuntimeV2": + if self.kind is RuntimeKindV2.PROCESS and self.evidence_seam is None: + raise ValueError("evidence_seam_required: kind=process") + if self.kind is RuntimeKindV2.COMPOSE and not self.document: + raise ValueError("compose_runtime_requires_document") + if self.kind is not RuntimeKindV2.COMPOSE and self.document is not None: + raise ValueError("document_only_for_compose") + if self.document: + _safe_relative(self.document) + return self + + +# --- §2b processes ------------------------------------------------------------------------- + + +class ProcessKind(str, Enum): + MANAGED = "managed" + SOURCE = "source" + + +class ManagedEngine(str, Enum): + POSTGRES = "postgres" + REDIS = "redis" + RABBITMQ = "rabbitmq" + + +class ProcessUser(str, Enum): + """The snapshot's fixed, bundle-assignable users (§0). `svc-control` runs ALK itself and is + never a process's own user.""" + + SVC_AGENT = "svc-agent" + SVC_TOOLS = "svc-tools" + SVC_DATA = "svc-data" + + +class SecretPurpose(str, Enum): + TARGET_PROVIDER = "target_provider" + SOURCE_CHECKOUT = "source_checkout" + + +class StartedCheck(BaseModel): + model_config = ConfigDict(extra="forbid") + + port: int | None = Field(default=None, ge=1, le=65535) + log_marker: str | None = None + timeout_seconds: float = Field(default=30.0, gt=0) + + @model_validator(mode="after") + def _exactly_one_probe(self) -> "StartedCheck": + if (self.port is None) == (self.log_marker is None): + raise ValueError("started_check_requires_exactly_one_of_port_or_log_marker") + return self + + +class ManagedProcess(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal[ProcessKind.MANAGED] = ProcessKind.MANAGED + engine: ManagedEngine + version: str = Field(min_length=1) + user: ProcessUser + depends_on: list[str] = Field(default_factory=list) + + +class SourceProcess(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal[ProcessKind.SOURCE] = ProcessKind.SOURCE + working_directory: str + build_commands: list[list[str]] = Field(default_factory=list) + run_command: list[str] = Field(min_length=1) + environment: dict[str, str] = Field(default_factory=dict) + build_environment: dict[str, str] | None = None + fixed_port: int | None = Field(default=None, ge=1, le=65535) + started_check: StartedCheck | None = None + secret_purposes: list[SecretPurpose] = Field(default_factory=list) + user: ProcessUser + depends_on: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _shape(self) -> "SourceProcess": + _safe_relative(self.working_directory) + for step in self.build_commands: + if not step: + raise ValueError("build_command_step_empty") + return self + + +ProcessEntry = Annotated[Union[ManagedProcess, SourceProcess], Field(discriminator="kind")] + + +# --- §2c seed ------------------------------------------------------------------------------ + + +class BaselineStrategy(str, Enum): + TEMPLATE_DATABASE = "template_database" + DATADIR_COPY = "datadir_copy" + EMPTY = "empty" + + +# The §2b catalog table. The engine a store answers to is the *capability's* protocol (resolved in +# the root validator, where `capabilities` is in scope) — a store entry carries no `engine` field +# of its own, and the sentinel's shape is a proof of that engine, not a second source for it. +_ENGINE_STRATEGIES: dict[ManagedEngine, frozenset[BaselineStrategy]] = { + ManagedEngine.POSTGRES: frozenset( + {BaselineStrategy.TEMPLATE_DATABASE, BaselineStrategy.DATADIR_COPY} + ), + ManagedEngine.REDIS: frozenset({BaselineStrategy.DATADIR_COPY, BaselineStrategy.EMPTY}), + ManagedEngine.RABBITMQ: frozenset({BaselineStrategy.DATADIR_COPY}), +} + + +class StoreBaseline(BaseModel): + model_config = ConfigDict(extra="forbid") + + strategy: BaselineStrategy + inputs_digest: str + + @model_validator(mode="after") + def _digest_shape(self) -> "StoreBaseline": + if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.inputs_digest): + raise ValueError("inputs_digest_invalid") + return self + + +class Sentinel(BaseModel): + """A store's per-protocol read-only proof, per §2c: postgres `{query, expected}`, redis + `{key, expected}`, rabbitmq `{queue, expected_depth}` — exactly one shape, never a mix.""" + + model_config = ConfigDict(extra="forbid") + + query: str | None = None + expected: str | None = None + key: str | None = None + queue: str | None = None + expected_depth: int | None = Field(default=None, ge=0) + + @model_validator(mode="after") + def _one_protocol_shape(self) -> "Sentinel": + if self.implied_engine is None: + raise ValueError( + "sentinel_shape_invalid: expected exactly one of " + "postgres{query,expected}, redis{key,expected}, rabbitmq{queue,expected_depth}" + ) + return self + + @property + def implied_engine(self) -> ManagedEngine | None: + postgres = self.query is not None and self.expected is not None + redis = self.key is not None and self.expected is not None + rabbitmq = self.queue is not None and self.expected_depth is not None + shapes = [ + (postgres, ManagedEngine.POSTGRES, {self.key, self.queue, self.expected_depth}), + (redis, ManagedEngine.REDIS, {self.query, self.queue, self.expected_depth}), + (rabbitmq, ManagedEngine.RABBITMQ, {self.query, self.key, self.expected}), + ] + matched = [ + engine for present, engine, others in shapes if present and others == {None} + ] + return matched[0] if len(matched) == 1 else None + + +class StoreEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + + capability: str = Field(min_length=1) + migrations: list[str] = Field(default_factory=list) + seed_files: list[str] = Field(default_factory=list) + baseline: StoreBaseline + sentinel: Sentinel + + @model_validator(mode="after") + def _paths(self) -> "StoreEntry": + for relative_path in (*self.migrations, *self.seed_files): + _safe_relative(relative_path) + return self + + +class Seed(BaseModel): + model_config = ConfigDict(extra="forbid") + + stores: list[StoreEntry] = Field(default_factory=list) + + +# --- §2d capabilities, readiness, files, provenance ----------------------------------------- + + +class CapabilityV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + protocol: CapabilityProtocol + service: str = Field(min_length=1) + container_port: int | None = Field(default=None, ge=1, le=65535) + configuration_name: str | None = None + + +class ReadinessProbeV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + capability: str + path: str | None = None + timeout_seconds: float = Field(default=120.0, gt=0, le=1800) + interval_seconds: float = Field(default=1.0, gt=0, le=60) + + +class BundleFileV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + path: str + sha256: str + size: int = Field(ge=0) + + @model_validator(mode="after") + def _valid_path(self) -> "BundleFileV2": + _safe_relative(self.path) + if not re.fullmatch(r"[0-9a-f]{64}", self.sha256): + raise ValueError(f"file_sha256_invalid: {self.path}") + return self + + +class BundleProvenanceV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_kind: str + repository: str | None = None + commit: str | None = None + source_digest: str + generator: str = "fi.alk.harness" + generator_version: str = "1" + adopted_files: list[str] = Field(default_factory=list) + generated_files: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _valid_source_digest(self) -> "BundleProvenanceV2": + # Bare 64-hex, matching what `source_fingerprint` (v1's producer) actually emits — no + # `sha256:` prefix, unlike `digest`/`inputs_digest`. + if not re.fullmatch(r"[0-9a-f]{64}", self.source_digest): + raise ValueError(f"source_digest_invalid: {self.source_digest}") + return self + + +# --- manifest root --------------------------------------------------------------------------- + +_CAPABILITY_SLUG = re.compile(r"[a-z][a-z0-9_]*") + +# §2c: a store's engine is the engine behind its capability's protocol, not a field the store +# carries itself. Only postgres/redis/amqp capabilities can host a store at all (§2c); any other +# protocol on a store's capability is a producer error §2e is left to catch. +_STORE_ENGINE_BY_PROTOCOL: dict[CapabilityProtocol, ManagedEngine] = { + CapabilityProtocol.POSTGRES: ManagedEngine.POSTGRES, + CapabilityProtocol.REDIS: ManagedEngine.REDIS, + CapabilityProtocol.AMQP: ManagedEngine.RABBITMQ, +} + + +class EnvironmentBundleV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str + digest: str + name: str + runtime: BundleRuntimeV2 + processes: list[ProcessEntry] = Field(default_factory=list) + seed: Seed | None = None + capabilities: dict[str, CapabilityV2] = Field(default_factory=dict) + readiness: list[ReadinessProbeV2] = Field(default_factory=list) + files: list[BundleFileV2] = Field(default_factory=list) + provenance: BundleProvenanceV2 + metadata: dict[str, JsonValue] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_manifest(self) -> "EnvironmentBundleV2": + if self.schema_version != BUNDLE_V2_SCHEMA_VERSION: + raise ValueError(f"bundle_schema_unsupported: {self.schema_version}") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.digest): + raise ValueError("bundle_digest_invalid") + + if self.runtime.kind is RuntimeKindV2.PROCESS and not self.processes: + raise ValueError("processes_required: kind=process") + if self.runtime.kind is RuntimeKindV2.EXTERNAL and ( + self.processes or self.seed is not None + ): + raise ValueError("processes_and_seed_forbidden: kind=external") + + for slug in self.capabilities: + if not _CAPABILITY_SLUG.fullmatch(slug): + raise ValueError(f"capability_slug_invalid: {slug}") + + process_names = Counter(process.name for process in self.processes) + duplicated_names = sorted(name for name, count in process_names.items() if count > 1) + if duplicated_names: + raise ValueError("process_name_duplicate: " + ", ".join(duplicated_names)) + known_names = set(process_names) + + service_unresolved = { + slug: capability.service + for slug, capability in self.capabilities.items() + if capability.service not in known_names + } + if service_unresolved: + detail = ", ".join( + f"{slug}: {service}" for slug, service in sorted(service_unresolved.items()) + ) + raise ValueError(f"service_unresolved: {detail}") + + control_service = self.runtime.control_service + if control_service is not None and control_service not in known_names: + raise ValueError(f"control_service_unresolved: {control_service}") + + names_to_slugs: dict[str, list[str]] = {} + for slug, capability in self.capabilities.items(): + if capability.configuration_name: + names_to_slugs.setdefault(capability.configuration_name, []).append(slug) + duplicated = {name: slugs for name, slugs in names_to_slugs.items() if len(slugs) > 1} + if duplicated: + detail = ", ".join( + f"{name} ({', '.join(sorted(slugs))})" for name, slugs in sorted(duplicated.items()) + ) + raise ValueError(f"configuration_name_duplicate: {detail}") + + unresolved = { + probe.capability + for probe in self.readiness + if probe.capability not in self.capabilities + } + if self.seed is not None: + for store in self.seed.stores: + if store.capability not in self.capabilities: + unresolved.add(store.capability) + if unresolved: + raise ValueError("capability_unresolved: " + ", ".join(sorted(unresolved))) + + if self.seed is not None: + # Every store's capability resolved above, so its protocol is known — that protocol, + # not the sentinel's own shape, is the authoritative engine (§2c classifies stores by + # capability protocol; the sentinel only proves that engine, it doesn't select it). + for store in self.seed.stores: + engine = _STORE_ENGINE_BY_PROTOCOL.get(self.capabilities[store.capability].protocol) + if engine is None: + continue + if store.sentinel.implied_engine is not engine: + raise ValueError( + f"sentinel_shape_mismatch: {store.capability}: sentinel implies " + f"{store.sentinel.implied_engine.value}, capability protocol resolves to " + f"{engine.value}" + ) + if store.baseline.strategy not in _ENGINE_STRATEGIES[engine]: + raise ValueError( + f"seed_strategy_unsupported: {store.capability}: {engine.value} does not " + f"support {store.baseline.strategy.value}" + ) + + if self.seed is not None: + # A store names its capability directly (no placeholder to resolve), so this half of + # the configuration_name rule is decidable here; the process-`environment` half needs + # placeholder scanning against the closed `{{...}}` vocabulary, which is §2e's job. + missing_name = [ + store.capability + for store in self.seed.stores + if not self.capabilities[store.capability].configuration_name + ] + if missing_name: + raise ValueError( + "configuration_name_required: " + ", ".join(sorted(set(missing_name))) + ) + + # v1's whole-manifest resolved-secret guard (`bundle.py`), reapplied here rather than + # dropped: v2 newly carries free-form `environment`/`build_environment` dicts, exactly + # where a resolved credential lands if an authoring stage ever inlines one instead of + # routing it through `secret_purposes`. `secret_purposes` itself is excluded from the + # dump — its key matches the secret-field pattern (`secret_...`) but it holds purpose + # identifiers, never values, the same reasoning v1 exempts `secret_refs` under. + _reject_secret_values( + self.model_dump(exclude={"digest": True, "processes": {"__all__": {"secret_purposes"}}}) + ) + return self + + +# --- §2c inputs_digest ------------------------------------------------------------------------ + + +def compute_inputs_digest( + root: str | Path, + migrations: Sequence[str], + seed_files: Sequence[str], + *, + engine: ManagedEngine, + version: str, +) -> str: + """The byte-exact `seed.stores[].baseline.inputs_digest` construction from §2c. + + sha256 over, for each file in ``migrations`` then ``seed_files`` **in listed order** (never + sorted — order is part of the identity, since migrations must apply in sequence), + ``\\n\\n``, followed by ``:\\n``. + Runs at authoring time, against ``migrations``/``seed_files`` as bundle-relative paths under + ``root`` (the bundle staging root) — the same paths the sealed manifest records, so the digest + is reproducible from the bundle's own field values. ``engine``/``version`` must be the store's + engine's own declared `ManagedProcess.engine`/`.version`, verbatim. + """ + root = Path(root) + digest = hashlib.sha256() + for relative_path in (*migrations, *seed_files): + _safe_relative(relative_path) + content = (root / relative_path).read_bytes() + digest.update(relative_path.encode("utf-8")) + digest.update(b"\n") + digest.update(str(len(content)).encode("utf-8")) + digest.update(b"\n") + digest.update(content) + digest.update(f"{engine.value}:{version}\n".encode("utf-8")) + return "sha256:" + digest.hexdigest() + + +def load_bundle_v2(path: str | Path) -> EnvironmentBundleV2: + """Parse and validate one `futureagi.environment-bundle.v2` manifest. + + ``path`` is either the manifest file itself or the bundle directory containing it. Schema + version is checked before full model validation runs, so a `…bundle.v1` manifest — or + anything else — is named explicitly rather than failing on an unrelated field. + """ + path = Path(path).expanduser() + target = path if path.is_file() else path / BUNDLE_V2_MANIFEST + if not target.is_file(): + raise BundleV2Error(f"bundle_manifest_missing: {target}") + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BundleV2Error(f"bundle_manifest_invalid: {exc}") from exc + if not isinstance(raw, dict): + raise BundleV2Error("bundle_manifest_invalid: not a JSON object") + schema_version = raw.get("schema_version") + if schema_version != BUNDLE_V2_SCHEMA_VERSION: + raise BundleV2Error(f"bundle_schema_unsupported: {schema_version!r}") + try: + return EnvironmentBundleV2.model_validate(raw) + except ValidationError as exc: + raise BundleV2Error(f"bundle_manifest_invalid: {exc}") from exc + + +__all__ = [ + "BUNDLE_V2_MANIFEST", + "BUNDLE_V2_SCHEMA_VERSION", + "BaselineStrategy", + "BundleFileV2", + "BundleProvenanceV2", + "BundleRuntimeV2", + "BundleV2Error", + "CapabilityV2", + "EnvironmentBundleV2", + "EvidenceSeam", + "ManagedEngine", + "ManagedProcess", + "ProcessKind", + "ProcessUser", + "ReadinessProbeV2", + "RuntimeKindV2", + "Seed", + "SecretPurpose", + "Sentinel", + "SourceProcess", + "StartedCheck", + "StoreBaseline", + "StoreEntry", + "compute_inputs_digest", + "load_bundle_v2", +] diff --git a/src/fi/alk/harness/checks.py b/src/fi/alk/harness/checks.py index e5dc01ad..71ed82f2 100644 --- a/src/fi/alk/harness/checks.py +++ b/src/fi/alk/harness/checks.py @@ -69,7 +69,7 @@ def run_check( return Outcome( name, False, - f"the check raised {type(failed).__name__}: {failed}", + f"the check raised {type(failed).__name__}: {str(failed)[:200]}", broken=True, ) @@ -81,8 +81,8 @@ def run_check( return Outcome( name, False, - f"the check returned {type(said).__name__} {said!r}; a check returns a sentence " - "naming what is wrong, or None when it held.", + f"the check returned {type(said).__name__} {repr(said)[:200]}; a check returns a " + "sentence naming what is wrong, or None when it held.", broken=True, ) return Outcome(name, False, said) diff --git a/src/fi/alk/harness/folder.py b/src/fi/alk/harness/folder.py index d9864126..fa6fec93 100644 --- a/src/fi/alk/harness/folder.py +++ b/src/fi/alk/harness/folder.py @@ -54,8 +54,9 @@ if len(_sys.argv) > 2: _calls = [_Call(**_one) for _one in _json.loads(_Path(_sys.argv[2]).read_text())] _said = check(_world, _calls) - print("held" if _said is None else f"FAILED: {_said}") - raise SystemExit(0 if _said is None else 1) + _held = _said is None or _said is True or (isinstance(_said, str) and not _said.strip()) + print("held" if _held else f"FAILED: {_said}") + raise SystemExit(0 if _held else 1) """ @@ -113,8 +114,8 @@ def _run(source: str, name: str, entry: str, *args: Any) -> Outcome: # into a precondition sentence, so any of them is our mistake rather than the world's. return Outcome( False, - f"{name} returned {type(said).__name__} {said!r}. Return the sentence naming what is " - "missing, or None if it holds.", + f"{name} returned {type(said).__name__} {repr(said)[:200]}. Return the sentence " + "naming what is missing, or None if it holds.", broken=True, ) return Outcome(False, str(said)) diff --git a/tests/harness/test_bundle_v2.py b/tests/harness/test_bundle_v2.py new file mode 100644 index 00000000..db44a18e --- /dev/null +++ b/tests/harness/test_bundle_v2.py @@ -0,0 +1,439 @@ +"""`futureagi.environment-bundle.v2` model validation, per `hosted-execution-seams.md` v1.6 §2. + +Two lanes: the spec's own §2a/§2b/§2c example structures, transcribed here and proven to parse +(the model-layer accept side), against the rejections the model is responsible for on its own — +wrong schema version, an unknown process kind, a store with no sentinel, a sentinel shape that +disagrees with its capability's protocol, a strategy the capability's protocol-implied engine does +not support, unresolved `service`/`control_service`/capability references, a duplicate process +name, and a resolved secret value anywhere in the manifest. `compute_inputs_digest` is checked +against a hand-computed vector, not by calling back into itself. + +Rules that need a repo checkout, the job the bundle will run under, or the §2e checklist +(`compose_not_hosted`, `engine_unsupported`, `no_sql_store`, `depends_on` cycles, placeholder +vocabulary, reserved-name scanning) belong to Phase 4's preflight and are out of scope here. +""" + +from __future__ import annotations + +import hashlib +import json + +import pytest +from pydantic import ValidationError + +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + BundleRuntimeV2, + BundleV2Error, + EnvironmentBundleV2, + ManagedEngine, + ManagedProcess, + SourceProcess, + StoreEntry, + compute_inputs_digest, + load_bundle_v2, +) + +# --- §2a/§2b/§2c: the spec's own examples, transcribed verbatim ----------------------------- + +RUNTIME_EXAMPLE = {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"} + +POSTGRES_PROCESS_EXAMPLE = { + "name": "postgres", + "kind": "managed", + "engine": "postgres", + "version": "16", + "user": "svc-data", + "depends_on": [], +} + +TOOLS_API_PROCESS_EXAMPLE = { + "name": "tools-api", + "kind": "source", + "working_directory": "services/tools-api", + "build_commands": [["npm", "ci"]], + "run_command": ["node", "server.js"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "PORT": "{{PORT_tools-api}}", + "TMPDIR": "{{WORLD_DIR}}", + }, + "secret_purposes": [], + "user": "svc-tools", + "depends_on": ["postgres"], +} + +AGENT_PROCESS_EXAMPLE = { + "name": "agent", + "kind": "source", + "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent/agent.py", "start"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "TOOLS_API_URL": "{{TOOLS_API_URL}}", + "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], + "user": "svc-agent", + "depends_on": ["postgres", "tools-api"], +} + +# §2b catalog-table engines beyond postgres, for the capability-protocol/sentinel/strategy pairing +# tests (F1) — a store's engine now comes from its capability's protocol, so these tests need a +# real managed process of the matching engine for the capability's `service` to resolve to. +REDIS_PROCESS_EXAMPLE = { + "name": "cache", + "kind": "managed", + "engine": "redis", + "version": "7", + "user": "svc-data", + "depends_on": [], +} + +RABBITMQ_PROCESS_EXAMPLE = { + "name": "queue", + "kind": "managed", + "engine": "rabbitmq", + "version": "3.13", + "user": "svc-data", + "depends_on": [], +} + +# The spec's own `<64-hex>` placeholder, filled with a real hex string — the example is only +# illustrating the digest's shape, not a value to reproduce. +SEED_STORE_EXAMPLE = { + "capability": "database", + "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + "baseline": { + "strategy": "template_database", + "inputs_digest": "sha256:" + "a" * 64, + }, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "12"}, +} + +FULL_MANIFEST_EXAMPLE = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "demo", + "runtime": RUNTIME_EXAMPLE, + "processes": [POSTGRES_PROCESS_EXAMPLE, TOOLS_API_PROCESS_EXAMPLE, AGENT_PROCESS_EXAMPLE], + "seed": {"stores": [SEED_STORE_EXAMPLE]}, + "capabilities": { + "database": { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "DATABASE_URL", + }, + "tools": { + "protocol": "http", + "service": "tools-api", + "configuration_name": "TOOLS_API_URL", + }, + }, + "readiness": [{"capability": "tools", "path": "/healthz"}], + "files": [{"path": "db/schema.sql", "sha256": "b" * 64, "size": 10}], + "provenance": { + "source_kind": "repository", + "repository": "org/repo", + "commit": "a" * 40, + # Bare 64-hex — what v1's `source_fingerprint` producer actually emits, unlike the + # `sha256:`-prefixed `digest`/`inputs_digest`. + "source_digest": "c" * 64, + }, +} + + +def test_the_runtime_example_from_2a_is_accepted() -> None: + runtime = BundleRuntimeV2.model_validate(RUNTIME_EXAMPLE) + assert runtime.kind.value == "process" + assert runtime.evidence_seam.value == "http_tool" + + +def test_the_managed_process_example_from_2b_is_accepted() -> None: + process = ManagedProcess.model_validate(POSTGRES_PROCESS_EXAMPLE) + assert process.engine.value == "postgres" + assert process.version == "16" + + +@pytest.mark.parametrize( + "example", [TOOLS_API_PROCESS_EXAMPLE, AGENT_PROCESS_EXAMPLE], ids=["tools-api", "agent"] +) +def test_the_source_process_examples_from_2b_are_accepted(example: dict) -> None: + process = SourceProcess.model_validate(example) + assert process.run_command + assert process.working_directory == example["working_directory"] + + +def test_the_seed_store_example_from_2c_is_accepted() -> None: + store = StoreEntry.model_validate(SEED_STORE_EXAMPLE) + assert store.baseline.strategy.value == "template_database" + assert store.sentinel.implied_engine is ManagedEngine.POSTGRES + + +def test_the_full_manifest_assembled_from_the_spec_examples_is_accepted() -> None: + bundle = EnvironmentBundleV2.model_validate(FULL_MANIFEST_EXAMPLE) + assert bundle.name == "demo" + assert [process.name for process in bundle.processes] == ["postgres", "tools-api", "agent"] + assert bundle.seed is not None and bundle.seed.stores[0].capability == "database" + + +# --- rejections the model is responsible for on its own ------------------------------------- + + +def test_wrong_schema_version_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "schema_version": "futureagi.environment-bundle.v1"} + with pytest.raises(ValidationError, match="bundle_schema_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_unknown_process_kind_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [{**POSTGRES_PROCESS_EXAMPLE, "kind": "container"}], + } + with pytest.raises(ValidationError, match="union_tag_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_with_no_sentinel_is_rejected() -> None: + incomplete = {key: value for key, value in SEED_STORE_EXAMPLE.items() if key != "sentinel"} + with pytest.raises(ValidationError, match="sentinel"): + StoreEntry.model_validate(incomplete) + + +# The engine a store answers to is now the engine behind its *capability's protocol* (F1), decided +# in the root validator — so these cases are built as full manifests, not bare `StoreEntry`s: the +# capability, its protocol, and a real process for `service` to resolve to (F5) all have to be in +# scope together for the rule to fire at all. +@pytest.mark.parametrize( + "process,protocol,sentinel,strategy", + [ + # redis's sentinel shape only pairs with datadir_copy or empty (§2b's catalog table). + (REDIS_PROCESS_EXAMPLE, "redis", {"key": "warm", "expected": "1"}, "template_database"), + # rabbitmq's sentinel shape only pairs with datadir_copy. + (RABBITMQ_PROCESS_EXAMPLE, "amqp", {"queue": "jobs", "expected_depth": 0}, "empty"), + ], + ids=["redis", "rabbitmq"], +) +def test_a_strategy_the_capabilitys_engine_does_not_support_is_rejected( + process: dict, protocol: str, sentinel: dict, strategy: str +) -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], process], + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "store": { + "protocol": protocol, + "service": process["name"], + "configuration_name": "STORE_URL", + }, + }, + "seed": { + "stores": [ + SEED_STORE_EXAMPLE, + { + "capability": "store", + "baseline": {"strategy": strategy, "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": sentinel, + }, + ] + }, + } + with pytest.raises(ValidationError, match="seed_strategy_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_redis_capability_with_a_postgres_shaped_sentinel_is_rejected() -> None: + """The capability's protocol decides the engine, not the sentinel's own shape — a redis + capability paired with a postgres-shaped sentinel is a shape mismatch even though the sentinel + is internally well-formed and the strategy is one postgres would have accepted.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": {**FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "protocol": "redis"}, + }, + } + with pytest.raises(ValidationError, match="sentinel_shape_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_postgres_capability_with_a_typod_redis_shaped_sentinel_is_rejected() -> None: + """A postgres capability whose sentinel was typo'd to redis's `{key, expected}` shape is + rejected naming the sentinel mismatch, not `seed_strategy_unsupported` against an engine + (`redis`) the bundle never declared — the bug this replaces would have named the wrong one.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": {"stores": [{**SEED_STORE_EXAMPLE, "sentinel": {"key": "warm", "expected": "1"}}]}, + } + with pytest.raises(ValidationError, match="sentinel_shape_mismatch") as exc_info: + EnvironmentBundleV2.model_validate(manifest) + assert "seed_strategy_unsupported" not in str(exc_info.value) + + +def test_a_sentinel_mixing_two_protocol_shapes_is_rejected() -> None: + store = { + "capability": "database", + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1", "key": "also-set"}, + } + with pytest.raises(ValidationError, match="sentinel_shape_invalid"): + StoreEntry.model_validate(store) + + +def test_unknown_field_on_a_process_entry_is_rejected() -> None: + with pytest.raises(ValidationError): + ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "mounts": ["/data"]}) + + +# --- §2a/§2b/§2d rules the model owns on its own, exercised at manifest scope ---------------- + + +def test_a_process_runtime_without_evidence_seam_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "runtime": {"kind": "process", "control_service": "agent"}} + with pytest.raises(ValidationError, match="evidence_seam_required"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_process_runtime_with_no_processes_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "processes": []} + with pytest.raises(ValidationError, match="processes_required"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_an_external_runtime_carrying_processes_or_seed_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "runtime": {"kind": "external"}} + with pytest.raises(ValidationError, match="processes_and_seed_forbidden"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_readiness_probe_naming_an_unresolved_capability_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "readiness": [{"capability": "does-not-exist"}]} + with pytest.raises(ValidationError, match="capability_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_duplicate_process_name_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], dict(POSTGRES_PROCESS_EXAMPLE)], + } + with pytest.raises(ValidationError, match="process_name_duplicate"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_capability_service_naming_an_unknown_process_is_rejected() -> None: + """The underscore/hyphen slip finding 5 names directly: a `service` of `tools_api` against a + process actually named `tools-api` must be caught here, not surface as an infrastructure + failure when the provisioner finds nothing to attach the capability to.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "tools": {**FULL_MANIFEST_EXAMPLE["capabilities"]["tools"], "service": "tools_api"}, + }, + } + with pytest.raises(ValidationError, match="service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_an_unresolved_control_service_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {**RUNTIME_EXAMPLE, "control_service": "not-a-process"}, + } + with pytest.raises(ValidationError, match="control_service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_resolved_secret_value_in_process_environment_is_rejected() -> None: + """v1's manifest-level guard, reapplied: `environment`/`build_environment` are new in v2 and + are exactly where a resolved credential lands if an authoring stage inlines one instead of + routing it through `secret_purposes`.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + TOOLS_API_PROCESS_EXAMPLE, + { + **AGENT_PROCESS_EXAMPLE, + "environment": { + **AGENT_PROCESS_EXAMPLE["environment"], + "STRIPE_API_KEY": "sk_live_x", + }, + }, + ], + } + with pytest.raises(ValidationError, match="resolved_secret_forbidden"): + EnvironmentBundleV2.model_validate(manifest) + + +# --- §2c inputs_digest: byte-exact construction, checked against a hand-computed vector ----- + + +def test_compute_inputs_digest_matches_a_hand_computed_vector(tmp_path) -> None: + (tmp_path / "db").mkdir() + (tmp_path / "café").mkdir() + schema = b"CREATE TABLE riders (id int);" + # Multi-byte content pins content_length as a byte count, not a character count — reading via + # read_text() + len(text) would compute 8 here instead of 9 and still hash to something, just + # not this. Path with a non-ASCII segment pins the path encoding as utf-8, not latin-1. + seed = "-- café\n".encode("utf-8") + notes = b"-- see docs" + (tmp_path / "db" / "schema.sql").write_bytes(schema) + (tmp_path / "db" / "seed.sql").write_bytes(seed) + (tmp_path / "café" / "notes.sql").write_bytes(notes) + + # Built from §2c's own words, not by calling the helper: for each file in migrations then + # seed_files, in listed order, \n\n, then + # :\n. + expected = hashlib.sha256() + expected.update(b"db/schema.sql\n" + str(len(schema)).encode() + b"\n" + schema) + expected.update(b"db/seed.sql\n" + str(len(seed)).encode() + b"\n" + seed) + notes_path = "café/notes.sql".encode("utf-8") + expected.update(notes_path + b"\n" + str(len(notes)).encode() + b"\n" + notes) + expected.update(b"postgres:16\n") + + got = compute_inputs_digest( + tmp_path, + ["db/schema.sql"], + ["db/seed.sql", "café/notes.sql"], + engine=ManagedEngine.POSTGRES, + version="16", + ) + assert got == "sha256:" + expected.hexdigest() + + +def test_compute_inputs_digest_is_order_sensitive_not_sorted(tmp_path) -> None: + """Two migrations reversed must hash differently — order is part of the identity, since + migrations apply in sequence, not in whatever order sorting would put them in.""" + (tmp_path / "a.sql").write_bytes(b"A") + (tmp_path / "b.sql").write_bytes(b"B") + + forward = compute_inputs_digest( + tmp_path, ["a.sql", "b.sql"], [], engine=ManagedEngine.POSTGRES, version="16" + ) + backward = compute_inputs_digest( + tmp_path, ["b.sql", "a.sql"], [], engine=ManagedEngine.POSTGRES, version="16" + ) + assert forward != backward + + +# --- load_bundle_v2 -------------------------------------------------------------------------- + + +def test_load_bundle_v2_rejects_a_v1_manifest_with_a_typed_error(tmp_path) -> None: + (tmp_path / "manifest.json").write_text( + json.dumps({"schema_version": "futureagi.environment-bundle.v1"}) + ) + with pytest.raises(BundleV2Error, match="bundle_schema_unsupported"): + load_bundle_v2(tmp_path) + + +def test_load_bundle_v2_parses_a_valid_manifest_from_its_directory(tmp_path) -> None: + (tmp_path / "manifest.json").write_text(json.dumps(FULL_MANIFEST_EXAMPLE)) + bundle = load_bundle_v2(tmp_path) + assert bundle.name == "demo" diff --git a/tests/harness/test_runner_conventions.py b/tests/harness/test_runner_conventions.py index ee9984de..f5444e19 100644 --- a/tests/harness/test_runner_conventions.py +++ b/tests/harness/test_runner_conventions.py @@ -83,7 +83,7 @@ def test_setup_returning_any_other_value_stays_advisory(returning: str) -> None: """The widened rule is keyed on entry == "ready", not on the value alone — setup() keeps its own untouched convention for every one of these values too.""" outcome = _run(f"def setup(world):\n return {returning}\n", "s/setup.py", "setup", None) - assert not outcome.broken + assert not outcome.broken and not outcome.ok def test_check_ready_reports_bare_false_as_broken() -> None: diff --git a/tests/harness/test_world_handle.py b/tests/harness/test_world_handle.py index bba11686..47bc0683 100644 --- a/tests/harness/test_world_handle.py +++ b/tests/harness/test_world_handle.py @@ -268,6 +268,19 @@ def test_bare_state_raises_when_every_table_is_over_the_cap() -> None: world.state() +def test_bare_state_names_both_causes_when_nothing_survives_the_exclusion() -> None: + """The widened message covers the mixed case too: one baseline table over the cap plus one + table the agent created since construction — excluding both would leave the snapshot `{}`, + so the raised message must name both `orders` (over-cap) and `late_table` (never measured), + not just whichever cause the wording happens to lead with.""" + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP + 1}) + world._store.tables["late_table"] = [{"id": 1}] + with pytest.raises(WorldStateTooLarge) as raised: + world.state() + assert "never measured" in str(raised.value) + assert "late_table" in str(raised.value) and "orders" in str(raised.value) + + def test_a_table_missing_from_the_baseline_dict_is_unavailable_not_under_cap() -> None: """A missing entry used to default to a row count of 0 and read as under the cap; the cap cannot be decided for a table nobody measured at freeze. The check runs once, at From f617f0292cbdeec2dfbf4923e65138c76a2cc78d Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 04:49:03 +0530 Subject: [PATCH 04/20] feat(harness): pre-provision preflight and the bundle seal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The complete §2e checklist as typed preflight errors — digest and filesystem verification scoped to the real directory, secret material scanning, unknown-field translation, placeholder vocabulary, secret purposes both directions, depends_on graph, engine catalog, reserved names, inputs-digest verification, and the compose gate ahead of all file-level checks so the dispositive code wins. seal_bundle_v2() is the normative §2d digest producer, pinned by a hand-computed byte vector. Model layer gains the user-assignment, engine-agreement, and reserved-configuration-name rules. Signed-off-by: khushalsonawat --- src/fi/alk/harness/bundle_v2.py | 158 ++++- src/fi/alk/harness/process_preflight.py | 596 +++++++++++++++++++ tests/harness/test_bundle_v2.py | 353 ++++++++++- tests/harness/test_process_preflight.py | 740 ++++++++++++++++++++++++ 4 files changed, 1820 insertions(+), 27 deletions(-) create mode 100644 src/fi/alk/harness/process_preflight.py create mode 100644 tests/harness/test_process_preflight.py diff --git a/src/fi/alk/harness/bundle_v2.py b/src/fi/alk/harness/bundle_v2.py index f5e77d3d..710666e0 100644 --- a/src/fi/alk/harness/bundle_v2.py +++ b/src/fi/alk/harness/bundle_v2.py @@ -1,4 +1,5 @@ -"""`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape. +"""`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape (`hosted-execution- +seams.md` v1.7). v1 (`bundle.py`) describes a `command`-per-service compose world and embeds the repository source. v2 describes `/work/source` as already present and a job that starts plain processes on @@ -26,7 +27,7 @@ from collections import Counter from enum import Enum from pathlib import Path -from typing import Annotated, Literal, Sequence, Union +from typing import Annotated, Any, Literal, Sequence, Union from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator @@ -254,6 +255,13 @@ class Seed(BaseModel): # --- §2d capabilities, readiness, files, provenance ----------------------------------------- +# §2b's closed placeholder vocabulary, mirrored here (not imported from `process_preflight.py`, +# which imports this module) so a `configuration_name` can never shadow a builtin token — the +# reverse dependency direction is preflight -> model, not model -> preflight. +_RESERVED_CONFIGURATION_NAMES = {"WORLD_INDEX", "WORLD_DIR", "DB_NAME"} +_RESERVED_CONFIGURATION_PREFIX = re.compile(r"^(PORT|HOST)_") + + class CapabilityV2(BaseModel): model_config = ConfigDict(extra="forbid") @@ -262,6 +270,18 @@ class CapabilityV2(BaseModel): container_port: int | None = Field(default=None, ge=1, le=65535) configuration_name: str | None = None + @model_validator(mode="after") + def _configuration_name_not_reserved(self) -> "CapabilityV2": + # A `configuration_name` colliding with a fixed placeholder or a `{{PORT_/HOST_}}` prefix + # would render the builtin token instead of this capability's address, with no error and + # no way for the producer to spell the intended value (F8, p4-round1-review). + name = self.configuration_name + if name and ( + name in _RESERVED_CONFIGURATION_NAMES or _RESERVED_CONFIGURATION_PREFIX.match(name) + ): + raise ValueError(f"configuration_name_reserved: {name}") + return self + class ReadinessProbeV2(BaseModel): model_config = ConfigDict(extra="forbid") @@ -360,21 +380,55 @@ def _validate_manifest(self) -> "EnvironmentBundleV2": if duplicated_names: raise ValueError("process_name_duplicate: " + ", ".join(duplicated_names)) known_names = set(process_names) + processes_by_name = {process.name: process for process in self.processes} + + # B3 (p3-round2-review): only `kind: process` has a `processes` array to resolve against — + # `external` omits `processes` entirely (§2a) and `compose` addresses services through its + # own `document`, not this array. Gating here, rather than by emptying `known_names`, + # keeps the duplicate-name check above meaningful for every runtime kind. + if self.runtime.kind is RuntimeKindV2.PROCESS: + service_unresolved = { + slug: capability.service + for slug, capability in self.capabilities.items() + if capability.service not in known_names + } + if service_unresolved: + detail = ", ".join( + f"{slug}: {service}" for slug, service in sorted(service_unresolved.items()) + ) + raise ValueError(f"service_unresolved: {detail}") + + control_service = self.runtime.control_service + if control_service is not None and control_service not in known_names: + raise ValueError(f"control_service_unresolved: {control_service}") + if control_service is not None and isinstance( + processes_by_name[control_service], ManagedProcess + ): + # §2a: control_service is the agent-side service the world handle and evidence + # seam attach to — a datastore in that role is incoherent, and would otherwise + # silently resolve and take svc-agent below (N9, p4-round2-review). + raise ValueError( + f"control_service_unresolved: {control_service} is a managed engine, not a " + "source process" + ) - service_unresolved = { - slug: capability.service - for slug, capability in self.capabilities.items() - if capability.service not in known_names - } - if service_unresolved: - detail = ", ".join( - f"{slug}: {service}" for slug, service in sorted(service_unresolved.items()) - ) - raise ValueError(f"service_unresolved: {detail}") - - control_service = self.runtime.control_service - if control_service is not None and control_service not in known_names: - raise ValueError(f"control_service_unresolved: {control_service}") + # §2b/§0 (v1.6): the snapshot's SERVICE users are assigned by role, not authored — + # the control service gets svc-agent, every other source process svc-tools, every + # managed engine svc-data. Decidable from the manifest's own fields alone once + # `control_service` is resolved, which is why it lands here rather than in preflight + # (F5, p4-round1-review). + for process in self.processes: + if isinstance(process, ManagedProcess): + expected_user = ProcessUser.SVC_DATA + elif process.name == control_service: + expected_user = ProcessUser.SVC_AGENT + else: + expected_user = ProcessUser.SVC_TOOLS + if process.user is not expected_user: + raise ValueError( + f"user_assignment_invalid: {process.name} must be " + f"{expected_user.value}, got {process.user.value}" + ) names_to_slugs: dict[str, list[str]] = {} for slug, capability in self.capabilities.items(): @@ -399,14 +453,47 @@ def _validate_manifest(self) -> "EnvironmentBundleV2": if unresolved: raise ValueError("capability_unresolved: " + ", ".join(sorted(unresolved))) + # B1 (p3-round2-review): a capability's *declared* protocol can disagree with the process + # actually backing it. F19 (p4-round1-review) widened this from "only capabilities with a + # seed store" to every capability whose protocol names a managed engine — a redis + # capability with no store entry at all (used only for a `{{...}}` address, never seeded) + # was previously never checked, and could point `service` at a postgres process silently. + for slug, capability in self.capabilities.items(): + engine = _STORE_ENGINE_BY_PROTOCOL.get(capability.protocol) + if engine is None: + continue + backing = processes_by_name.get(capability.service) + if isinstance(backing, ManagedProcess) and backing.engine is not engine: + raise ValueError( + f"capability_engine_mismatch: {slug}: protocol {capability.protocol.value} " + f"resolves to {engine.value}, but {capability.service} is a " + f"{backing.engine.value} process" + ) + if self.seed is not None: # Every store's capability resolved above, so its protocol is known — that protocol, # not the sentinel's own shape, is the authoritative engine (§2c classifies stores by # capability protocol; the sentinel only proves that engine, it doesn't select it). for store in self.seed.stores: - engine = _STORE_ENGINE_BY_PROTOCOL.get(self.capabilities[store.capability].protocol) + capability = self.capabilities[store.capability] + engine = _STORE_ENGINE_BY_PROTOCOL.get(capability.protocol) if engine is None: - continue + # B2 (p3-round2-review): a store on a capability outside the three protocols + # this module knows how to seed (http, mongodb, ...) has no engine to check + # its sentinel/strategy against — a producer error, not a silent pass-through. + raise ValueError( + f"store_protocol_unsupported: {store.capability}: protocol " + f"{capability.protocol.value} cannot host a seed store" + ) + backing = processes_by_name.get(capability.service) + if not isinstance(backing, ManagedProcess): + # F19 (p4-round1-review): a store on a capability backed by a `SourceProcess` + # has no managed engine to migrate or seed at all — the all-capabilities + # engine pass above only fires for a *wrong* managed engine, not a missing one. + raise ValueError( + f"store_service_not_managed: {store.capability}: service " + f"{capability.service!r} is not a managed engine" + ) if store.sentinel.implied_engine is not engine: raise ValueError( f"sentinel_shape_mismatch: {store.capability}: sentinel implies " @@ -480,6 +567,40 @@ def compute_inputs_digest( return "sha256:" + digest.hexdigest() +# --- §2d bundle digest ------------------------------------------------------------------------ + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode( + "utf-8" + ) + + +def seal_bundle_v2(manifest: EnvironmentBundleV2) -> str: + """The byte-exact `digest` construction from §2d (v1.7) — the single normative + implementation; producers call this, never reimplement it. + + sha256 over the canonical dump of the manifest with ``digest`` and ``files`` removed, then for + each ``files[]`` record, IN LISTED ORDER, the canonical dump of ``{path, sha256, size}`` + prefixed by its byte length as 8 bytes big-endian. "Canonical" = ``json.dumps(..., + sort_keys=True, separators=(",", ":"), ensure_ascii=False)``, both times. Operates on + `BundleFileV2` and `EnvironmentBundleV2.model_dump(mode="json")` directly — v1's `BundleFile` + never enters this construction, so a field added to v1's model cannot silently rekey a v2 + bundle's digest (F4, p4-round1-review). The hash covers the NORMALIZED dump, so adding an + optional field to `EnvironmentBundleV2` re-keys every previously sealed bundle — sealer and + verifier must ship together, which is exactly why there is only one implementation. + """ + core = manifest.model_dump(mode="json") + core.pop("digest", None) + core.pop("files", None) + digest = hashlib.sha256(_canonical_json(core)) + for record in manifest.files: + encoded = _canonical_json(record.model_dump(mode="json")) + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return "sha256:" + digest.hexdigest() + + def load_bundle_v2(path: str | Path) -> EnvironmentBundleV2: """Parse and validate one `futureagi.environment-bundle.v2` manifest. @@ -532,4 +653,5 @@ def load_bundle_v2(path: str | Path) -> EnvironmentBundleV2: "StoreEntry", "compute_inputs_digest", "load_bundle_v2", + "seal_bundle_v2", ] diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py new file mode 100644 index 00000000..e774f625 --- /dev/null +++ b/src/fi/alk/harness/process_preflight.py @@ -0,0 +1,596 @@ +"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.7 — as a single gate the +in-sandbox provisioner runs before starting anything. + +`bundle_v2.py` validates everything decidable from the manifest's own field values alone; this +module covers what its docstring names as deferred: the bundle's actual files on disk (digest and +per-file hashes, symlinks, path escapes, secret content), the pydantic `extra="forbid"` -> +`unknown_field` translation, and every rule that needs the job the bundle will run under +(placeholder vocabulary, secret purposes against the job's `secret_refs`, the `depends_on` graph, +the engine catalog, `seed_missing`, `inputs_digest` verification, reserved-name content scanning, +`no_sql_store`, and resource sanity). `seed_strategy_unsupported`, `sentinel_shape_mismatch`, +`capability_unresolved`, `configuration_name_duplicate`, `user_assignment_invalid`, and +`capability_engine_mismatch` are already enforced by the model layer and are not repeated here. + +A missing interpreter (§0, v1.7) is a BUILD-time failure, not a preflight one — no manifest field +carries an interpreter demand, so this module has nothing to check and does not attempt to. + +`preflight_bundle` runs the checklist in the contract's own order and raises on the first +violation, never a crash — every failure is a `PreflightError` carrying a code from §2e's +failure-code table (v1.7). The caller (the provisioner) is responsible for mapping that into a +FAILED terminal state with `FailureDomain.ENVIRONMENT` in `HarnessStage.VALIDATING_ENVIRONMENT`, +per §2e's closing rule. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path, PurePosixPath + +from pydantic import ValidationError + +from .artifacts import _SECRET_CONTENT, _SECRET_FILES +from .bundle import CapabilityProtocol +from .bundle_v2 import ( + BUNDLE_V2_MANIFEST, + BUNDLE_V2_SCHEMA_VERSION, + BundleFileV2, + EnvironmentBundleV2, + ManagedEngine, + ManagedProcess, + RuntimeKindV2, + SecretPurpose, + SourceProcess, + compute_inputs_digest, + seal_bundle_v2, +) + +_SECRET_PURPOSE_VALUES = {member.value for member in SecretPurpose} + + +class PreflightError(RuntimeError): + """A §2e checklist rule rejected the bundle. + + ``code`` is one of §2e's failure-code table (v1.7): "contract-rule" codes, each named by a + numbered checklist item's prose, and "mechanical" codes for plumbing failures the contract + describes but does not formalize as a rule (a missing bundle file, an out-of-range + ``parallelism``). Every code this module raises is in that table. + """ + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +_SECRET_SUFFIXES = {".pem", ".key", ".p12", ".pfx"} + +# §2b's catalog table. `ManagedEngine` already closes which *engines* exist; this closes which +# *version* of each is the one the snapshot actually ships. +_ENGINE_CATALOG_VERSION: dict[ManagedEngine, str] = { + ManagedEngine.POSTGRES: "16", + ManagedEngine.REDIS: "7", + ManagedEngine.RABBITMQ: "3.13", +} + +# §0 (v1.7): a repo needing an interpreter the snapshot lacks fails at BUILD time, reported +# `runtime_unsupported` there — not here. The manifest carries no interpreter-demand field (the +# source tree isn't embedded in the bundle, so preflight can't see `.python-version`/`engines` +# even if it wanted to), so this module has no interpreter check to run. + +# §2c: migrations/seeds must not create these — checked as a source-content scan, not a manifest +# field, since the identifier lives inside SQL/scripts the model layer never parses. +# `re.IGNORECASE`: postgres folds an unquoted identifier to lower case, so `CREATE TABLE +# _ALK_CONFORMANCE` creates the reserved table under its lower-case name — case-insensitive +# matching is the only way to catch that (F9, p4-round1-review). This is slightly over-broad for +# redis/rabbitmq, whose names are case-sensitive, but over-broad on a reserved-name check is the +# safe direction. Known false-positive surface, left as-is (documented rather than fixed): the scan +# reads whole file bytes with no lexical awareness beyond stripped `--`/`/* */` comments below, so +# a quoted string literal containing the reserved name (e.g. as inserted *data*) still trips it. +# The stripping below is a false-NEGATIVE surface in the opposite direction, equally lexer-free and +# equally left as-is: a `--` or `/*` inside a string literal (not a comment) deletes real content +# up to the next line-end or `*/`, which can delete a reserved-name definition that follows it on +# the same statement (N7, p4-round2-review). +_RESERVED_NAME = "_alk_conformance" +_RESERVED_NAME_PATTERN = re.compile( + r"(? None: + """Run the complete §2e checklist against a sealed v2 bundle directory, in the contract's own + numbered order. Raises ``PreflightError`` on the first violation; returns ``None`` when clean. + + ``manifest`` is the already-parsed model the caller obtained from ``load_bundle_v2`` — item 4 + (the pydantic ``extra_forbidden`` -> ``unknown_field`` translation bundle_v2's own docstring + defers here) is implemented by re-validating the bytes on disk, which is also where this + function's own read of ``manifest.json`` for step 1 comes from; a caller that already trusts + ``manifest`` still gets a genuine check that the file backing it hasn't drifted since. + + ``secret_refs`` maps each job secret alias to its ``SecretRef.purpose`` value (§1) — item 5's + ``secret_unclaimed``/``secret_missing`` pair needs it and the contract's own entrypoint + signature (§2e's charter) does not carry it. Required, not optional: §4's provider port hands + the provisioner ``work_directory``, and `/work/job.json` is readable from it, so every real + caller has the job's resolved refs — there is no legitimate caller that cannot supply this. + Pass ``{}`` explicitly for a job with no secret refs at all, rather than omitting the argument: + an optional default silently both under- and over-enforced the check it exists for (F2, + p4-round1-review), which required-and-explicit closes. Every value must be a ``SecretPurpose`` + value; anything else raises ``ValueError`` immediately, before any bundle content is checked. + """ + bundle_dir = Path(bundle_dir) + for alias, purpose in secret_refs.items(): + # `isinstance` first: §1's raw `agent.secret_refs` shape is `{alias: {manager, key, + # version, purpose}}`, a dict — an unhashable value would otherwise raise TypeError against + # the `in` check below instead of the ValueError this docstring promises (N8, p4-round2- + # review). + if not isinstance(purpose, str) or purpose not in _SECRET_PURPOSE_VALUES: + raise ValueError(f"secret_refs[{alias!r}] = {purpose!r} is not a SecretPurpose value") + + if manifest.runtime.kind is RuntimeKindV2.COMPOSE: + # §2a: "a hosted job with kind: compose fails preflight" — not one of §2e's seven numbered + # items, so ahead of item 1 rather than slotted between them: every item below assumes + # v2's processes/seed shape, which a compose bundle need not carry, and a compose bundle's + # own files (its document, e.g.) carry no obligation to be exhaustively listed in files[] + # the way a hosted bundle's do — checking file-listing first mis-reported that case as + # bundle_file_unlisted instead of compose_not_hosted (N1, p4-round2-review). + raise PreflightError("compose_not_hosted", "kind: compose is not a legal hosted runtime") + + files = _verify_digest(bundle_dir, manifest) # 1 + walked_files = _verify_path_safety(bundle_dir, files) # 2 + _scan_bundle_files_for_secrets(bundle_dir, walked_files) # 3 + _verify_unknown_fields(bundle_dir, manifest) # 4 + + if manifest.runtime.kind is RuntimeKindV2.PROCESS: + _verify_placeholder_vocabulary(manifest) # 5 + _verify_no_root_build_commands(manifest) # 5 (§2a) + _verify_secret_purposes(manifest, secret_refs) # 5 + _verify_depends_on(manifest) # 5 + _verify_engine_catalog(manifest) # 5 + _verify_seed_missing(manifest) # 5 / §2c + _verify_reserved_names(bundle_dir, manifest) # 5 + _verify_seed_files_on_disk_and_listed(bundle_dir, manifest, files) # 5 + + _verify_no_sql_store(manifest) # 6 + _verify_resource_sanity(manifest, parallelism=parallelism) # 7 + + +# --- item 1: digest verification ------------------------------------------------------------- + + +def _verify_digest(bundle_dir: Path, manifest: EnvironmentBundleV2) -> list[BundleFileV2]: + root = bundle_dir.resolve() + try: + raw = json.loads((root / BUNDLE_V2_MANIFEST).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError("bundle_manifest_invalid", str(exc)) from exc + on_disk_schema_version = raw.get("schema_version") if isinstance(raw, dict) else None + if on_disk_schema_version != BUNDLE_V2_SCHEMA_VERSION: + # §2e item 1 opens with "schema_version is …bundle.v2" — checked here, at item 1, + # rather than left to surface three items late through item 4's re-validation fallback + # (F11, p4-round1-review). + raise PreflightError("bundle_schema_unsupported", str(on_disk_schema_version)) + for record in manifest.files: + path = root / record.path + if not path.is_file(): + raise PreflightError("bundle_file_missing", record.path) + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + if digest.hexdigest() != record.sha256 or size != record.size: + raise PreflightError("bundle_file_changed", record.path) + recomputed = seal_bundle_v2(manifest) + if recomputed != manifest.digest: + raise PreflightError( + "bundle_digest_mismatch", f"expected {manifest.digest}, computed {recomputed}" + ) + return manifest.files + + +# --- item 2: path safety on the filesystem itself ---------------------------------------------- + + +def _verify_path_safety(bundle_dir: Path, files: list[BundleFileV2]) -> list[Path]: + """The model already rejects unsafe strings in `files[].path` (`_safe_relative`); this walks + the actual filesystem, which a string check cannot: a symlinked directory can make an + innocent-looking relative path resolve outside the bundle root. + + Every non-directory entry except the bundle root's own `manifest.json` must be recorded in + `files[]` (`bundle_file_unlisted`) — a file physically present but never listed was invisible + to both the digest check above and the secret scan that follows, which is exactly what let an + unlisted `.env` through undetected (F1, p4-round1-review). The `manifest.json` exemption is by + exact root path, not by basename (F10, p4-round1-review): a nested `db/manifest.json` gets no + special treatment, only `bundle_dir/manifest.json` itself. The exemption covers only the + listing check, not the symlink check — a symlinked root `manifest.json` would otherwise be + waved through here and then read straight through by `_verify_digest`/`_verify_unknown_fields`, + the very item whose job is to stop path escapes (N3, p4-round2-review). + + Returns the walked file paths so the secret scan (item 3) can run against what the filesystem + actually contains rather than against `files[]` again. + """ + root = bundle_dir.resolve() + manifest_path = root / BUNDLE_V2_MANIFEST + listed = {record.path for record in files} + walked: list[Path] = [] + for entry in root.rglob("*"): + if entry.is_symlink(): + raise PreflightError("bundle_symlink_forbidden", str(entry.relative_to(root))) + if entry == manifest_path: + continue + if entry.is_dir(): + continue + relative = entry.relative_to(root).as_posix() + if relative not in listed: + raise PreflightError("bundle_file_unlisted", relative) + walked.append(entry) + return walked + + +# --- item 3: secret material in the bundle's own files ------------------------------------------ + + +def _scan_bundle_files_for_secrets(bundle_dir: Path, walked_files: list[Path]) -> None: + """Reuses `artifacts.py`'s own file-name and content secret scan unchanged — the same + high-entropy-token regexes and credential-file-name set this codebase already applies to + sealed run artifacts, applied here to a sealed bundle's files instead. + + Scoped to every file item 2's filesystem walk actually found, not to `files[]` (F1, + p4-round1-review) — an unlisted secret file is already rejected by item 2's own + `bundle_file_unlisted` check, but this scan must not depend on that running first to be + correct on its own terms. + """ + root = bundle_dir.resolve() + for path in walked_files: + relative = path.relative_to(root).as_posix() + posix_path = PurePosixPath(relative) + if posix_path.name in _SECRET_FILES or posix_path.suffix.lower() in _SECRET_SUFFIXES: + raise PreflightError("secret_in_bundle", f"{relative}: forbidden secret-shaped file") + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + if any(pattern.search(chunk) for pattern in _SECRET_CONTENT): + raise PreflightError( + "secret_in_bundle", f"{relative}: high-entropy secret-scan hit" + ) + + +# --- item 4: unknown-field translation ---------------------------------------------------------- + + +def _verify_unknown_fields(bundle_dir: Path, manifest: EnvironmentBundleV2) -> None: + target = bundle_dir / BUNDLE_V2_MANIFEST + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError("bundle_manifest_invalid", str(exc)) from exc + try: + revalidated = EnvironmentBundleV2.model_validate(raw) + except ValidationError as exc: + raise _translate_validation_error(exc) from exc + if revalidated.model_dump(mode="json") != manifest.model_dump(mode="json"): + # Re-validating catches drift that makes the file *invalid*; it says nothing about drift + # that leaves it valid (a changed `run_command`, a flipped `user`) unless the two dumps are + # actually compared (F12, p4-round1-review). + raise PreflightError( + "bundle_manifest_drifted", "manifest.json on disk no longer matches manifest argument" + ) + + +def _translate_validation_error(exc: ValidationError) -> PreflightError: + """§2b: "unknown keys in a process entry are a preflight error (`unknown_field`)" — the model + layer's docstring defers this exact translation here, since pydantic's own `extra_forbidden` + carries no contract vocabulary of its own. Every other model-layer rejection already embeds + its own snake_case code as the leading token of its message (see `bundle_v2.py`'s + `model_validator`s); that code is preserved rather than collapsed into a generic one. + """ + for error in exc.errors(): + if error.get("type") == "extra_forbidden": + location = ".".join(str(part) for part in error["loc"]) + return PreflightError("unknown_field", f"{location}: unknown field") + # `(?::|$)`, not just `:` (F13, p4-round1-review): a bare code with no trailing detail (e.g. + # `bundle_digest_invalid`) is the entire message, with nothing after it to require a colon + # before. Scans every error, not just the first, since pydantic's own ordering is not the + # contract's priority — the first message that yields a recognizable code wins. + for error in exc.errors(): + message = str(error.get("msg", "")) + matched = re.match(r"(?:Value error, )?([a-z][a-z0-9_]*)(?::|$)", message) + if matched: + return PreflightError(matched.group(1), message) + return PreflightError("bundle_manifest_invalid", str(exc)) + + +# --- item 5: everything the model layer needs the job or the files for ------------------------ + + +def _verify_placeholder_vocabulary(manifest: EnvironmentBundleV2) -> None: + """§2b's closed `{{...}}` vocabulary, checked in `environment`. `build_environment` takes NO + placeholders at all (§2b) — any `{{...}}` match there is rejected outright, never resolved + against the vocabulary below (F6, p4-round1-review). + + `{{}}` can only ever resolve to a capability whose `configuration_name` + is non-null — a capability left null is therefore structurally unreachable by any placeholder, + which is what makes this scan also enforce §2d's "non-null whenever referenced by any process + `environment`... entry" without a second pass. When the unmatched token is exactly a declared + capability's slug and that capability's `configuration_name` is null, the real problem is the + missing name, not the token — reported `capability_unresolved` naming the capability, rather + than the generic `unknown_placeholder` every other unmatched token gets (F15, p4-round1-review; + a deliberate resolution — §2d names no other string a producer could have meant). + """ + known_names = {process.name for process in manifest.processes} + known_configuration_names = { + capability.configuration_name + for capability in manifest.capabilities.values() + if capability.configuration_name + } + unresolved_capability_slugs = { + slug + for slug, capability in manifest.capabilities.items() + if not capability.configuration_name + } + for process in manifest.processes: + if not isinstance(process, SourceProcess): + continue + for key, value in (process.build_environment or {}).items(): + match = _PLACEHOLDER.search(value) + if match: + raise PreflightError( + "unknown_placeholder", + f"{process.name}.build_environment.{key}: {{{{{match.group(1)}}}}} — " + "build_environment takes no placeholders", + ) + for key, value in process.environment.items(): + for match in _PLACEHOLDER.finditer(value): + token = match.group(1) + if token in _FIXED_PLACEHOLDERS: + continue + named = _NAMED_PLACEHOLDER.match(token) + if named: + _, name = named.groups() + if name in known_names: + continue + raise PreflightError( + "unknown_placeholder", + f"{process.name}.environment.{key}: {{{{{token}}}}} names an unknown " + "process", + ) + if token in known_configuration_names: + continue + if token in unresolved_capability_slugs: + raise PreflightError( + "capability_unresolved", + f"{process.name}.environment.{key}: {{{{{token}}}}} names capability " + f"{token!r}, which has no configuration_name", + ) + raise PreflightError( + "unknown_placeholder", + f"{process.name}.environment.{key}: {{{{{token}}}}} is not in the closed " + "placeholder vocabulary", + ) + + +def _verify_no_root_build_commands(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, SourceProcess): + continue + for step in process.build_commands: + if step[0] in _ROOT_BUILD_COMMANDS or "sudo" in step: + raise PreflightError( + "build_requires_root", f"{process.name}: build step {step!r} requires root" + ) + + +def _verify_secret_purposes(manifest: EnvironmentBundleV2, secret_refs: dict[str, str]) -> None: + """§2b: both directions, scoped to `target_provider` only — `source_checkout` and any other + gateway-only purpose never crosses into the guest (§0 step 3) and has nothing to claim here.""" + ref_has_target_provider = any( + purpose == SecretPurpose.TARGET_PROVIDER.value for purpose in secret_refs.values() + ) + process_claims_target_provider = any( + SecretPurpose.TARGET_PROVIDER in process.secret_purposes + for process in manifest.processes + if isinstance(process, SourceProcess) + ) + if ref_has_target_provider and not process_claims_target_provider: + raise PreflightError( + "secret_unclaimed", "a target_provider secret ref is not listed by any process" + ) + if process_claims_target_provider and not ref_has_target_provider: + raise PreflightError( + "secret_missing", "a process lists secret_purposes: target_provider but the job " + "supplies no such ref" + ) + + +def _verify_depends_on(manifest: EnvironmentBundleV2) -> None: + graph = {process.name: list(process.depends_on) for process in manifest.processes} + for name, deps in graph.items(): + unknown = sorted(dep for dep in deps if dep not in graph) + if unknown: + raise PreflightError( + "depends_on_unresolved", + f"{name} depends_on unknown process(es): {', '.join(unknown)}", + ) + + unvisited, in_progress, done = 0, 1, 2 + state = {name: unvisited for name in graph} + + def visit(name: str, stack: list[str]) -> None: + state[name] = in_progress + stack.append(name) + for dep in graph[name]: + if state[dep] == in_progress: + cycle = stack[stack.index(dep):] + [dep] + raise PreflightError("depends_on_cycle", " -> ".join(cycle)) + if state[dep] == unvisited: + visit(dep, stack) + stack.pop() + state[name] = done + + for name in sorted(graph): + if state[name] == unvisited: + visit(name, []) + + +def _verify_engine_catalog(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + pinned = _ENGINE_CATALOG_VERSION[process.engine] + if process.version != pinned: + raise PreflightError( + "engine_unsupported", + f"{process.name}: {process.engine.value} {process.version} is not supported; " + f"the snapshot ships {process.engine.value} {pinned}", + ) + + +def _verify_seed_missing(manifest: EnvironmentBundleV2) -> None: + covered = {store.capability for store in (manifest.seed.stores if manifest.seed else [])} + missing = sorted( + slug + for slug, capability in manifest.capabilities.items() + if capability.protocol is CapabilityProtocol.POSTGRES and slug not in covered + ) + if missing: + raise PreflightError( + "seed_missing", + "postgres-protocol capability with no store entry: " + ", ".join(missing), + ) + + +def _verify_reserved_names(bundle_dir: Path, manifest: EnvironmentBundleV2) -> None: + if manifest.seed is None: + return + root = bundle_dir.resolve() + for store in manifest.seed.stores: + for relative_path in (*store.migrations, *store.seed_files): + path = root / relative_path + if not path.is_file(): + continue # reported by `_verify_seed_files_on_disk_and_listed` + text = path.read_text(encoding="utf-8", errors="replace") + # Strip `--`-to-EOL and `/* ... */` comments before scanning (F9, p4-round1-review) — + # a generated seed file's own note about the reservation ("-- never create + # _alk_conformance here") would otherwise trip the scan on prose, not on an identifier + # it defines. Quoted string literals containing the name as *data* remain a known + # false-positive surface: the scan has no lexer, only comment-stripping. + code = _SQL_BLOCK_COMMENT.sub("", _SQL_LINE_COMMENT.sub("", text)) + if _RESERVED_NAME_PATTERN.search(code): + raise PreflightError( + "reserved_name", + f"{relative_path} defines the reserved conformance-canary identifier " + f"{_RESERVED_NAME!r}", + ) + + +def _verify_seed_files_on_disk_and_listed( + bundle_dir: Path, manifest: EnvironmentBundleV2, files: list[BundleFileV2] +) -> None: + """Digest verification (item 1) already guarantees every ``files[]``-listed path exists, so a + path missing from disk entirely is ``seed_file_missing`` regardless of whether it was ever + listed. ``seed_file_unlisted`` stays here as a second, store-scoped statement of the same + "listed" rule item 2's own walk now enforces bundle-wide (F1, p4-round1-review) — through + `preflight_bundle`'s full sequence item 2's ``bundle_file_unlisted`` always fires first for any + file the walk visits; the root ``manifest.json`` is exempt from that walk, so a store path + naming it still reaches here (N2, p4-round2-review). + + Once every migration/seed file for a store is confirmed present and listed, its recorded + ``inputs_digest`` is recomputed and compared (F14, p4-round1-review; §2c makes it the baseline + identity attempt-retry reuse trusts absolutely, and nothing else on either side of the seam + ever validated it). ``engine``/``version`` come from the store's capability's own backing + ``ManagedProcess`` — guaranteed to exist by `bundle_v2`'s ``store_service_not_managed`` check; + a non-``ManagedProcess`` backing here would mean that guarantee broke, raised as a typed + ``PreflightError`` rather than asserted, since this module's charter is a rejection on every + path, never a crash (N8, p4-round2-review). + """ + if manifest.seed is None: + return + listed = {record.path for record in files} + root = bundle_dir.resolve() + processes_by_name = {process.name: process for process in manifest.processes} + for store in manifest.seed.stores: + for relative_path in (*store.migrations, *store.seed_files): + if not (root / relative_path).is_file(): + raise PreflightError("seed_file_missing", f"{relative_path} does not exist on disk") + if relative_path not in listed: + raise PreflightError( + "seed_file_unlisted", f"{relative_path} is not listed in files[]" + ) + capability = manifest.capabilities[store.capability] + engine_process = processes_by_name[capability.service] + if not isinstance(engine_process, ManagedProcess): + raise PreflightError( + "store_service_not_managed", + f"{store.capability}: service {capability.service!r} is not a managed engine", + ) + recomputed = compute_inputs_digest( + root, + store.migrations, + store.seed_files, + engine=engine_process.engine, + version=engine_process.version, + ) + if recomputed != store.baseline.inputs_digest: + raise PreflightError( + "inputs_digest_mismatch", + f"{store.capability}: expected {store.baseline.inputs_digest}, computed " + f"{recomputed}", + ) + + +# --- item 6: no_sql_store ------------------------------------------------------------------------ + + +def _verify_no_sql_store(manifest: EnvironmentBundleV2) -> None: + if manifest.runtime.kind is not RuntimeKindV2.PROCESS: + return + if not any( + capability.protocol is CapabilityProtocol.POSTGRES + for capability in manifest.capabilities.values() + ): + raise PreflightError( + "no_sql_store", "kind: process requires at least one postgres-protocol capability" + ) + + +# --- item 7: resource sanity ---------------------------------------------------------------------- + + +def _verify_resource_sanity(manifest: EnvironmentBundleV2, *, parallelism: int) -> None: + if len(manifest.processes) > _MAX_PROCESSES: + raise PreflightError( + "process_count_exceeded", + f"{len(manifest.processes)} processes exceeds the {_MAX_PROCESSES} cap", + ) + if not (_MIN_PARALLELISM <= parallelism <= _MAX_PARALLELISM): + raise PreflightError( + "parallelism_out_of_range", + f"parallelism={parallelism} is outside {_MIN_PARALLELISM}..{_MAX_PARALLELISM}", + ) + + +__all__ = ["PreflightError", "preflight_bundle"] diff --git a/tests/harness/test_bundle_v2.py b/tests/harness/test_bundle_v2.py index db44a18e..7494e430 100644 --- a/tests/harness/test_bundle_v2.py +++ b/tests/harness/test_bundle_v2.py @@ -1,12 +1,14 @@ -"""`futureagi.environment-bundle.v2` model validation, per `hosted-execution-seams.md` v1.6 §2. +"""`futureagi.environment-bundle.v2` model validation, per `hosted-execution-seams.md` v1.7 §2. Two lanes: the spec's own §2a/§2b/§2c example structures, transcribed here and proven to parse (the model-layer accept side), against the rejections the model is responsible for on its own — -wrong schema version, an unknown process kind, a store with no sentinel, a sentinel shape that -disagrees with its capability's protocol, a strategy the capability's protocol-implied engine does -not support, unresolved `service`/`control_service`/capability references, a duplicate process -name, and a resolved secret value anywhere in the manifest. `compute_inputs_digest` is checked -against a hand-computed vector, not by calling back into itself. +wrong schema version, an unknown process kind, a store with no sentinel, a capability whose +protocol disagrees with the engine actually backing it, a sentinel shape that disagrees with its +capability's protocol, a strategy the capability's protocol-implied engine does not support, a +store on a capability protocol this module cannot seed at all, unresolved +`service`/`control_service`/capability references (scoped to `kind: process`, per p3-round2's B3), +a duplicate process name, and a resolved secret value anywhere in the manifest. +`compute_inputs_digest` is checked against a hand-computed vector, not by calling back into itself. Rules that need a repo checkout, the job the bundle will run under, or the §2e checklist (`compose_not_hosted`, `engine_unsupported`, `no_sql_store`, `depends_on` cycles, placeholder @@ -32,6 +34,7 @@ StoreEntry, compute_inputs_digest, load_bundle_v2, + seal_bundle_v2, ) # --- §2a/§2b/§2c: the spec's own examples, transcribed verbatim ----------------------------- @@ -246,21 +249,131 @@ def test_a_strategy_the_capabilitys_engine_does_not_support_is_rejected( EnvironmentBundleV2.model_validate(manifest) +def test_postgres_excludes_the_empty_strategy() -> None: + """The one row of §2b's catalog table with no dedicated param above: postgres supports only + `template_database`/`datadir_copy`, unlike redis and rabbitmq which both accept a store with + no baseline state at all. Reuses the base manifest's own postgres/database pairing rather than + adding a process, since postgres already backs its only seed store.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": { + "stores": [ + { + **SEED_STORE_EXAMPLE, + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + } + ] + }, + } + with pytest.raises(ValidationError, match="seed_strategy_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + def test_a_redis_capability_with_a_postgres_shaped_sentinel_is_rejected() -> None: """The capability's protocol decides the engine, not the sentinel's own shape — a redis capability paired with a postgres-shaped sentinel is a shape mismatch even though the sentinel - is internally well-formed and the strategy is one postgres would have accepted.""" + is internally well-formed and the strategy is one postgres would have accepted. The backing + process is genuinely redis (B1, p3-round2-review, requires protocol/engine agreement before + this check runs), so this stays isolated to the sentinel-shape question alone.""" manifest = { **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], REDIS_PROCESS_EXAMPLE], "capabilities": { **FULL_MANIFEST_EXAMPLE["capabilities"], - "database": {**FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "protocol": "redis"}, + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "protocol": "redis", + "service": "cache", + }, }, } with pytest.raises(ValidationError, match="sentinel_shape_mismatch"): EnvironmentBundleV2.model_validate(manifest) +def test_a_capability_protocol_that_disagrees_with_its_backing_process_engine_is_rejected() -> None: + """B1 (p3-round2-review): the sentinel and strategy below are both self-consistent with the + *declared* protocol (redis) — which is exactly what let this accept an invalid bundle before + the fix. The capability's `service` actually names the postgres process; only comparing the + protocol against `ManagedProcess.engine` catches it, since neither the sentinel shape nor the + strategy pairing is wrong on its own terms.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": {**FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "protocol": "redis"}, + }, + "seed": { + "stores": [ + { + "capability": "database", + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"key": "warm", "expected": "1"}, + } + ] + }, + } + with pytest.raises(ValidationError, match="capability_engine_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_on_an_unmapped_protocol_capability_is_rejected() -> None: + """B2 (p3-round2-review): before the fix, a store on a capability outside the + postgres/redis/amqp map made the engine lookup return `None` and the loop `continue`d, + skipping the sentinel and strategy checks entirely — a store on the `http` `tools` capability + validated with any sentinel and any strategy. It must now be rejected explicitly.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": { + "stores": [ + SEED_STORE_EXAMPLE, + { + "capability": "tools", + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"key": "anything", "expected": "1"}, + }, + ] + }, + } + with pytest.raises(ValidationError, match="store_protocol_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_capability_engine_mismatch_is_caught_even_with_no_seed_store_at_all() -> None: + """F19 (p4-round1-review): before the fix, `capability_engine_mismatch` only ran inside the + `seed.stores` loop — a capability with no store entry at all (used only for a `{{...}}` + address, never seeded) was never compared, and could silently point `service` at the wrong + engine.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "cache": { + "protocol": "redis", "service": "postgres", "configuration_name": "CACHE_URL" + }, + }, + } + with pytest.raises(ValidationError, match="capability_engine_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_whose_capability_service_is_a_source_process_is_rejected() -> None: + """F19 (p4-round1-review): a postgres-protocol store backed by a source process has no + managed engine to migrate or seed at all — previously accepted, since only a *wrong* managed + engine was checked, never a missing one.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "service": "tools-api" + }, + }, + } + with pytest.raises(ValidationError, match="store_service_not_managed"): + EnvironmentBundleV2.model_validate(manifest) + + def test_a_postgres_capability_with_a_typod_redis_shaped_sentinel_is_rejected() -> None: """A postgres capability whose sentinel was typo'd to redis's `{key, expected}` shape is rejected naming the sentinel mismatch, not `seed_strategy_unsupported` against an engine @@ -285,7 +398,7 @@ def test_a_sentinel_mixing_two_protocol_shapes_is_rejected() -> None: def test_unknown_field_on_a_process_entry_is_rejected() -> None: - with pytest.raises(ValidationError): + with pytest.raises(ValidationError, match="extra_forbidden"): ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "mounts": ["/data"]}) @@ -349,6 +462,98 @@ def test_an_unresolved_control_service_is_rejected() -> None: EnvironmentBundleV2.model_validate(manifest) +def test_a_control_service_resolving_to_a_managed_engine_is_rejected() -> None: + """N9 (p4-round2-review): `control_service` names the agent-side service the world handle and + evidence seam attach to (§2a) — a datastore in that role is incoherent. Before this check, the + `ManagedProcess` branch of the user-assignment loop below ran first and expected `svc-data` + for it, which `postgres` already has, so the bundle silently loaded.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {**RUNTIME_EXAMPLE, "control_service": "postgres"}, + } + with pytest.raises(ValidationError, match="control_service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +# --- §2b/§0 user assignment (F5, p4-round1-review): control service -> svc-agent, other source -> +# svc-tools, managed engine -> svc-data. Decidable from the manifest's own fields once +# `control_service` is resolved. -------------------------------------------------------------- + + +def test_the_control_service_process_with_the_wrong_user_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + TOOLS_API_PROCESS_EXAMPLE, + {**AGENT_PROCESS_EXAMPLE, "user": "svc-tools"}, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_non_control_source_process_claiming_svc_agent_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + {**TOOLS_API_PROCESS_EXAMPLE, "user": "svc-agent"}, + AGENT_PROCESS_EXAMPLE, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_managed_engine_with_a_non_svc_data_user_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + {**POSTGRES_PROCESS_EXAMPLE, "user": "svc-tools"}, + TOOLS_API_PROCESS_EXAMPLE, + AGENT_PROCESS_EXAMPLE, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +# --- §2d configuration_name must not collide with the fixed placeholder vocabulary (F8, +# p4-round1-review) — a collision would render the builtin token instead of the capability's +# address, with no error and no way to spell the intended value. ------------------------------- + + +def test_a_configuration_name_matching_a_fixed_placeholder_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "configuration_name": "WORLD_DIR", + }, + }, + } + with pytest.raises(ValidationError, match="configuration_name_reserved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_configuration_name_matching_the_port_host_prefix_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "configuration_name": "PORT_DB", + }, + }, + } + with pytest.raises(ValidationError, match="configuration_name_reserved"): + EnvironmentBundleV2.model_validate(manifest) + + def test_a_resolved_secret_value_in_process_environment_is_rejected() -> None: """v1's manifest-level guard, reapplied: `environment`/`build_environment` are new in v2 and are exactly where a resolved credential lands if an authoring stage inlines one instead of @@ -371,6 +576,66 @@ def test_a_resolved_secret_value_in_process_environment_is_rejected() -> None: EnvironmentBundleV2.model_validate(manifest) +def test_an_external_runtime_with_a_capability_is_accepted() -> None: + """B3 (p3-round2-review): `kind: external` has no `processes` array to resolve capability + `service`/`control_service` against (§2a omits `processes` for it entirely) — the F5 + resolution checks must not run for it, or every external bundle carrying any capability at all + would be unloadable, which §2a never intended.""" + manifest = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "external-demo", + "runtime": {"kind": "external"}, + "capabilities": { + "target": { + "protocol": "http", + "service": "customer-endpoint", + "configuration_name": "TARGET_URL", + }, + }, + "provenance": {"source_kind": "remote", "source_digest": "c" * 64}, + } + bundle = EnvironmentBundleV2.model_validate(manifest) + assert bundle.runtime.kind.value == "external" + assert bundle.capabilities["target"].service == "customer-endpoint" + + +# --- F7 / B6: the digest-shape regexes, exercised on their rejection side too ---------------- + + +def test_a_non_hex_file_sha256_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "files": [{"path": "db/schema.sql", "sha256": "not-hex", "size": 10}], + } + with pytest.raises(ValidationError, match="file_sha256_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_non_hex_source_digest_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "provenance": {**FULL_MANIFEST_EXAMPLE["provenance"], "source_digest": "not-hex"}, + } + with pytest.raises(ValidationError, match="source_digest_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_malformed_bundle_digest_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "digest": "not-a-digest"} + with pytest.raises(ValidationError, match="bundle_digest_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_malformed_inputs_digest_is_rejected() -> None: + store = { + **SEED_STORE_EXAMPLE, + "baseline": {"strategy": "template_database", "inputs_digest": "not-a-digest"}, + } + with pytest.raises(ValidationError, match="inputs_digest_invalid"): + StoreEntry.model_validate(store) + + # --- §2c inputs_digest: byte-exact construction, checked against a hand-computed vector ----- @@ -422,6 +687,76 @@ def test_compute_inputs_digest_is_order_sensitive_not_sorted(tmp_path) -> None: assert forward != backward +# --- §2d bundle digest: byte-exact construction, checked against a hand-computed vector (F4, +# p4-round1-review) — the single normative implementation, `seal_bundle_v2`, over `BundleFileV2` +# directly, with no v1 model conversion. ------------------------------------------------------- + + +def test_seal_bundle_v2_matches_a_hand_computed_vector() -> None: + """Built from §2d's own words (v1.7), not by calling back into `seal_bundle_v2` or into + `json.dumps` with the same settings: sha256 over the canonical dump of the manifest minus + `digest`/`files`, then for each `files[]` record, in listed order, the canonical dump of + `{path, sha256, size}` prefixed by its byte length as 8 bytes big-endian. "Canonical" = + `json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False)`. The literal core + JSON below is transcribed from a minimal, otherwise-empty manifest's own normalized field set + (a hand-verified constant, not a derived one) — a real, if trivial, `external`-kind bundle + with no processes, no seed, no capabilities, and two files. A non-ASCII `name` pins + `ensure_ascii=False` (an `ensure_ascii=True` implementation would diverge here), and listing + `b.txt` before `a.txt` pins IN LISTED ORDER against an implementation that silently sorts + (N6, p4-round2-review).""" + file_sha_a = "b" * 64 + file_sha_b = "c" * 64 + manifest = EnvironmentBundleV2.model_validate( + { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "café", + "runtime": {"kind": "external"}, + "provenance": {"source_kind": "remote", "source_digest": "a" * 64}, + "files": [ + {"path": "b.txt", "sha256": file_sha_b, "size": 5}, + {"path": "a.txt", "sha256": file_sha_a, "size": 3}, + ], + } + ) + + core_json = ( + '{"capabilities":{},"metadata":{},"name":"café","processes":[],"provenance":' + '{"adopted_files":[],"commit":null,"generated_files":[],"generator":' + '"fi.alk.harness","generator_version":"1","repository":null,"source_digest":"' + + "a" * 64 + + '","source_kind":"remote"},"readiness":[],"runtime":' + '{"control_service":null,"document":null,"evidence_seam":null,"kind":"external"},' + '"schema_version":"futureagi.environment-bundle.v2","seed":null}' + ) + record_b_json = '{"path":"b.txt","sha256":"' + file_sha_b + '","size":5}' + record_a_json = '{"path":"a.txt","sha256":"' + file_sha_a + '","size":3}' + + expected = hashlib.sha256(core_json.encode("utf-8")) + for record_json in (record_b_json, record_a_json): + encoded = record_json.encode("utf-8") + expected.update(len(encoded).to_bytes(8, "big")) + expected.update(encoded) + + assert seal_bundle_v2(manifest) == "sha256:" + expected.hexdigest() + + +def test_seal_bundle_v2_ignores_the_manifests_own_digest_field() -> None: + """§2d: the digest is computed over the manifest minus `digest` and `files` — a manifest + whose only difference is its own (placeholder or stale) `digest` value must seal identically.""" + body = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {"kind": "external"}, + "processes": [], + "seed": None, + "capabilities": {}, + "readiness": [], + } + a = EnvironmentBundleV2.model_validate({**body, "digest": "sha256:" + "0" * 64}) + b = EnvironmentBundleV2.model_validate({**body, "digest": "sha256:" + "1" * 64}) + assert seal_bundle_v2(a) == seal_bundle_v2(b) + + # --- load_bundle_v2 -------------------------------------------------------------------------- diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py new file mode 100644 index 00000000..7a7046bc --- /dev/null +++ b/tests/harness/test_process_preflight.py @@ -0,0 +1,740 @@ +"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.7. + +Every checklist item gets at least one rejection test carrying its named code, plus one clean +accept-lane run of the full checklist. Bundles are built as real directories under `tmp_path` with +real file bytes — the digest and per-file checks need actual content to hash, so a manifest built +purely in memory (as `test_bundle_v2.py` does) cannot exercise this module. No docker; every +managed-engine/process concept here is a manifest fact, never a running service. +""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any, Callable + +import pytest + +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + EnvironmentBundleV2, + ManagedEngine, + compute_inputs_digest, + seal_bundle_v2, +) +from fi.alk.harness.process_preflight import PreflightError, preflight_bundle + +SCHEMA_SQL = b"CREATE TABLE riders (id int);\n" +SEED_SQL = b"INSERT INTO riders VALUES (1);\n" + +TARGET_PROVIDER_REFS = {"LIVEKIT_API_KEY": "target_provider"} + + +def _base_manifest_body() -> dict[str, Any]: + """A minimal, otherwise-clean `kind: process` manifest: one postgres store behind a real + seeded capability, one source process that claims the job's only `target_provider` secret.""" + return { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", + "kind": "managed", + "engine": "postgres", + "version": "16", + "user": "svc-data", + "depends_on": [], + }, + { + "name": "agent", + "kind": "source", + "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], + "user": "svc-agent", + "depends_on": ["postgres"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "DATABASE_URL", + }, + }, + "readiness": [], + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64 + }, + "metadata": {}, + } + + +def _build_bundle( + root: Path, + *, + body_overrides: Callable[[dict[str, Any]], dict[str, Any]] | None = None, + extra_files: dict[str, bytes] | None = None, + unlisted_files: dict[str, bytes] | None = None, + include_seed: bool = True, +) -> EnvironmentBundleV2: + """Write a real, digest-consistent bundle directory and return its parsed manifest, sealed + through `seal_bundle_v2` — the module under test's own producer, not a second reimplementation + of it (the accept lane must prove this module agrees with a real producer, not just with + itself). ``unlisted_files`` writes real bytes to disk without ever hashing them into + ``files[]`` — the shape a producer bug or a stray leftover file takes, as opposed to + ``extra_files``, which is hashed in and fully listed.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL} + if extra_files: + file_contents.update(extra_files) + + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + + if include_seed: + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", + "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + + if body_overrides is not None: + body = body_overrides(body) + + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + + if unlisted_files: + for relative, content in unlisted_files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +# --- accept lane -------------------------------------------------------------------------------- + + +def test_a_clean_bundle_passes_the_whole_checklist(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + result = preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +# --- item 1: digest verification ---------------------------------------------------------------- + + +def test_a_file_whose_bytes_changed_after_sealing_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").write_bytes(b"MUTATED") + with pytest.raises(PreflightError, match="bundle_file_changed") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_file_changed" + + +def test_a_files_entry_missing_from_disk_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").unlink() + with pytest.raises(PreflightError, match="bundle_file_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_bundle_digest_that_does_not_match_the_recomputed_value_is_rejected( + tmp_path: Path, +) -> None: + manifest = _build_bundle(tmp_path) + tampered = manifest.model_copy(update={"digest": "sha256:" + "9" * 64}) + with pytest.raises(PreflightError, match="bundle_digest_mismatch"): + preflight_bundle(tmp_path, tampered, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 2: path safety on the filesystem -------------------------------------------------------- + + +def test_a_symlink_anywhere_under_the_bundle_is_rejected(tmp_path: Path) -> None: + """The model already forbids unsafe strings in `files[].path`; this is the filesystem-level + complement — a symlink is rejected even when it sits outside `files[]` entirely.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "evil-link").symlink_to(tmp_path / "db" / "schema.sql") + with pytest.raises(PreflightError, match="bundle_symlink_forbidden"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_root_manifest_json_itself_being_a_symlink_is_rejected(tmp_path: Path) -> None: + """N3 (p4-round2-review): the root `manifest.json` is exempt from the listing check (a + manifest cannot list itself), but that exemption must not extend to its symlink status — a + symlinked root manifest was previously read straight through by `_verify_digest` and + `_verify_unknown_fields`, the very item whose job is to stop path escapes.""" + manifest = _build_bundle(tmp_path) + outside = tmp_path.parent / f"{tmp_path.name}-manifest-target.json" + outside.write_bytes((tmp_path / "manifest.json").read_bytes()) + (tmp_path / "manifest.json").unlink() + (tmp_path / "manifest.json").symlink_to(outside) + with pytest.raises(PreflightError, match="bundle_symlink_forbidden"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_file_present_on_disk_but_not_listed_in_files_is_rejected(tmp_path: Path) -> None: + """F1 (p4-round1-review): a bundle directory containing a file the producer never hashed into + `files[]` was invisible to both the digest check and the secret scan — this is the case that + previously slipped a `.env` through undetected. `extra_files` (used by the item-3 tests below) + would have hashed it in; `unlisted_files` writes the same bytes without listing them at all.""" + manifest = _build_bundle(tmp_path, unlisted_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="bundle_file_unlisted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 3: secret material in the bundle's own files ------------------------------------------- + + +def test_a_dotenv_file_in_the_bundle_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path, extra_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_high_entropy_secret_content_in_a_bundle_file_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle( + tmp_path, extra_files={"db/notes.sql": b"-----BEGIN RSA PRIVATE KEY-----\nabc\n"} + ) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 4: unknown-field translation ------------------------------------------------------------ + + +def test_an_unknown_field_added_to_the_manifest_on_disk_is_rejected(tmp_path: Path) -> None: + """The `manifest` argument is already-parsed and therefore already clean; this proves the + translation fires against the bytes on disk, which is what a drifted or hand-edited + `manifest.json` would look like to a fresh `EnvironmentBundleV2.model_validate` call.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["mounts"] = ["/data"] + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="unknown_field") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "unknown_field" + + +def test_a_valid_but_drifted_manifest_on_disk_is_rejected(tmp_path: Path) -> None: + """F12 (p4-round1-review): re-validating the bytes on disk only catches drift that makes the + file *invalid* — this proves drift that leaves it valid (a changed `run_command`) is caught + too, by actually comparing the two dumps rather than discarding the re-validated one.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["processes"][1]["run_command"] = ["python", "other.py"] + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_manifest_drifted") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_manifest_drifted" + + +def test_an_unreadable_manifest_json_on_disk_is_rejected(tmp_path: Path) -> None: + """F18 (p4-round1-review): `bundle_manifest_invalid`'s parse-failure branch, untested before.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "manifest.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_manifest_invalid"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_non_extra_forbidden_model_rejection_surfaces_its_own_code(tmp_path: Path) -> None: + """F13/F18 (p4-round1-review): `_translate_validation_error`'s fallback path was untested. A + malformed on-disk `digest` (item 1 never reads it — only the `manifest` argument's own valid + digest and file hashes, which are untouched here) reaches item 4's re-validation and raises + `bundle_digest_invalid`, a bare code with no trailing colon at all — exactly the case the old + `:`-only regex flattened to the generic `bundle_manifest_invalid`.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["digest"] = "not-a-valid-digest" + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_digest_invalid") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_digest_invalid" + + +# --- §2a: compose is not a legal hosted runtime kind ---------------------------------------------- + + +def test_kind_compose_is_rejected(tmp_path: Path) -> None: + """N1 (p4-round2-review): the gate sits above item 1, so this fires regardless of whether + `compose.yaml` exists on disk or is listed in `files[]` — a compose bundle need carry neither + to be rejected, which is why this fixture writes no such file at all.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +# --- item 5: placeholder vocabulary, secret purposes, depends_on, engine catalog, interpreter, +# seed_missing, reserved names, build_requires_root, seed files on disk+listed --------------------- + + +def test_a_placeholder_outside_the_closed_vocabulary_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_port_placeholder_naming_an_unknown_process_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["PEER_PORT"] = "{{PORT_ghost-service}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_port_placeholder_with_an_empty_name_falls_through_to_unknown_placeholder( + tmp_path: Path, +) -> None: + """F18 (p4-round1-review): `(.+)` in `_NAMED_PLACEHOLDER` requires at least one character after + `PORT_`/`HOST_`, so `{{PORT_}}` does not match the named-placeholder pattern at all — it falls + through to the configuration-name set, misses, and is rejected as `unknown_placeholder`. A + plausible regression target if `(.+)` is ever relaxed to `(.*)`.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["EMPTY"] = "{{PORT_}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_token_naming_a_capability_with_no_configuration_name_is_capability_unresolved( + tmp_path: Path, +) -> None: + """F15 (p4-round1-review): a null `configuration_name` is structurally unspellable by any + placeholder — but when the unmatched token happens to be exactly a declared capability's own + slug, the real problem is the capability's missing name, not an unrecognized token, so this is + reported `capability_unresolved` naming the capability rather than the generic + `unknown_placeholder`.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["capabilities"]["cache"] = {"protocol": "http", "service": "postgres"} + body["processes"][1]["environment"]["FOO"] = "{{cache}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="capability_unresolved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_build_environment_rejects_any_placeholder_even_a_legal_one(tmp_path: Path) -> None: + """F6 (p4-round1-review): §2b's `build_environment` takes NO placeholders at all — this uses + `{{WORLD_DIR}}`, a perfectly legal token in `environment`, specifically because that is the + case a naive "scan against the same vocabulary" implementation would wave through.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_environment"] = {"TMPDIR": "{{WORLD_DIR}}"} + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_fixed_and_named_placeholders_are_accepted(tmp_path: Path) -> None: + """`{{WORLD_INDEX}}`/`{{WORLD_DIR}}`/`{{DB_NAME}}` need no lookup; `{{PORT_}}` and + `{{HOST_}}` need only a real process name — neither needs a capability.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["SCRATCH"] = "{{WORLD_DIR}}" + body["processes"][1]["environment"]["DB"] = "{{DB_NAME}}" + body["processes"][1]["environment"]["PEER"] = "{{HOST_postgres}}:{{PORT_postgres}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + result = preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +def test_a_build_command_requiring_root_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_commands"] = [["apt-get", "install", "-y", "ffmpeg"]] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="build_requires_root"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_build_command_with_sudo_anywhere_in_the_step_is_rejected(tmp_path: Path) -> None: + """F18 (p4-round1-review): only `apt-get` as argv[0] was exercised before — `"sudo" in step` + is exact-token list membership, not a substring match, so `sudo` appearing anywhere in the + argv list (not just as argv[0]) must trip it too.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_commands"] = [["scripts/setup.sh", "--with-sudo", "sudo"]] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="build_requires_root"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_target_provider_ref_no_process_lists_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["secret_purposes"] = [] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="secret_unclaimed"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_listed_secret_purpose_with_no_supplying_ref_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + with pytest.raises(PreflightError, match="secret_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +def test_omitting_secret_refs_is_a_typeerror(tmp_path: Path) -> None: + """F2 (p4-round1-review): `secret_refs` is required, not optional with an empty-dict default — + an optional default meant `secret_unclaimed` was structurally unreachable for any caller that + relied on it, while `secret_missing` fired on bundles the default should have let alone. This + pins the signature itself: omitting the argument must fail loudly, at the call site, rather + than silently reintroducing the empty-dict default.""" + manifest = _build_bundle(tmp_path) + with pytest.raises(TypeError): + preflight_bundle(tmp_path, manifest, parallelism=1) # type: ignore[call-arg] + + +def test_an_unrecognized_secret_purpose_value_is_rejected(tmp_path: Path) -> None: + """F2 (p4-round1-review), the signature's second defect: `secret_refs` values are only ever + compared by `==` against `SecretPurpose.TARGET_PROVIDER.value` — a typo'd purpose string (or + §1's raw per-alias dict shape, whose `purpose` values are dicts, not strings) silently never + matches, producing the same wrong verdict as the missing-validation default did. Validating + eagerly turns that into a loud, immediate `ValueError`.""" + manifest = _build_bundle(tmp_path) + with pytest.raises(ValueError, match="not a SecretPurpose"): + preflight_bundle( + tmp_path, manifest, parallelism=1, secret_refs={"LIVEKIT_API_KEY": "target-provider"} + ) + + +def test_a_depends_on_naming_an_unknown_process_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["depends_on"] = ["postgres", "ghost"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="depends_on_unresolved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_depends_on_cycle_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][0]["depends_on"] = ["agent"] + body["processes"][1]["depends_on"] = ["postgres"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="depends_on_cycle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_an_engine_version_outside_the_catalog_pin_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][0]["version"] = "15" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="engine_unsupported"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_postgres_capability_with_no_store_entry_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["capabilities"]["other_db"] = { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "OTHER_DB_URL", + } + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="seed_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def _reseal_schema_sql(tmp_path: Path, content: bytes) -> EnvironmentBundleV2: + """Rewrites `db/schema.sql` on disk and reseals through `seal_bundle_v2` — so digest + verification (item 1, which runs before item 5's content-scanning checks) passes on the + mutated content, isolating a test to the check the mutation is actually meant to exercise.""" + (tmp_path / "db" / "schema.sql").write_bytes(content) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + for record in raw["files"]: + if record["path"] == "db/schema.sql": + record["sha256"] = hashlib.sha256(content).hexdigest() + record["size"] = len(content) + raw["seed"]["stores"][0]["baseline"]["inputs_digest"] = compute_inputs_digest( + tmp_path, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + raw["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(raw) + raw["digest"] = seal_bundle_v2(normalized) + (tmp_path / "manifest.json").write_text(json.dumps(raw, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(raw) + + +def test_the_reserved_conformance_name_in_migration_content_is_rejected(tmp_path: Path) -> None: + _build_bundle(tmp_path) + manifest = _reseal_schema_sql(tmp_path, b"CREATE TABLE _alk_conformance (id int);\n") + with pytest.raises(PreflightError, match="reserved_name"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_reserved_conformance_name_is_matched_case_insensitively(tmp_path: Path) -> None: + """F9 (p4-round1-review): postgres folds an unquoted identifier to lower case, so + `CREATE TABLE _ALK_CONFORMANCE` creates the reserved table under its lower-case name — a + case-sensitive scan would miss exactly the evasion this exists to catch.""" + _build_bundle(tmp_path) + manifest = _reseal_schema_sql(tmp_path, b"CREATE TABLE _ALK_CONFORMANCE (id int);\n") + with pytest.raises(PreflightError, match="reserved_name"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_similarly_named_table_and_a_comment_mentioning_the_reserved_name_are_accepted( + tmp_path: Path, +) -> None: + """F9 (p4-round1-review) regression: `alk_conformance_backup` (no leading underscore, and a + different table) must not trip the scan — this is exactly what the lookarounds exist to keep + open. A `--` comment mentioning the reserved name as prose, not as an identifier it defines, + must not trip it either, now that comments are stripped before scanning.""" + _build_bundle(tmp_path) + manifest = _reseal_schema_sql( + tmp_path, + b"CREATE TABLE alk_conformance_backup (id int);\n" + b"-- never create _alk_conformance here\n", + ) + result = preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +def test_a_migration_path_not_listed_in_files_is_rejected(tmp_path: Path) -> None: + """`seed_file_unlisted` (item 5) and `bundle_file_unlisted` (item 2, F1 p4-round1-review) share + the same "on disk but not in files[]" condition — item 2's bundle-wide walk now runs first and + always wins this exact scenario, which is why this asserts the earlier-numbered item's code + rather than the one this test used to name before F1 closed item 2's gap.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["migrations"] = ["db/schema.sql", "db/extra.sql"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + # On disk, but never hashed into files[]. + (tmp_path / "db" / "extra.sql").write_bytes(b"-- extra\n") + with pytest.raises(PreflightError, match="bundle_file_unlisted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_seed_file_path_that_does_not_exist_on_disk_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["seed_files"] = ["db/seed.sql", "db/ghost.sql"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="seed_file_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_recorded_inputs_digest_that_does_not_match_the_seed_files_is_rejected( + tmp_path: Path, +) -> None: + """F14 (p4-round1-review): §2c makes `inputs_digest` the baseline identity attempt-retry reuse + trusts absolutely — nothing on either side of the seam validated it before. `engine`/`version` + come from the store's capability's own backing `ManagedProcess` (postgres/16 here).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["baseline"]["inputs_digest"] = "sha256:" + "f" * 64 + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="inputs_digest_mismatch"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 6: no_sql_store ------------------------------------------------------------------------ + + +def test_a_process_bundle_with_no_postgres_capability_is_rejected(tmp_path: Path) -> None: + """Keeps the `database` capability (so the `{{DATABASE_URL}}` placeholder in `agent`'s + environment still resolves) and only changes its protocol away from postgres, isolating this + from the placeholder-vocabulary check that would otherwise fire first.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["capabilities"]["database"]["protocol"] = "http" + body["seed"] = None + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate, include_seed=False) + with pytest.raises(PreflightError, match="no_sql_store"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 7: resource sanity --------------------------------------------------------------------- + + +def test_more_than_100_processes_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + extra = [ + { + "name": f"extra-{i}", + "kind": "managed", + "engine": "redis", + "version": "7", + "user": "svc-data", + "depends_on": [], + } + for i in range(100) + ] + body["processes"] = body["processes"] + extra + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="process_count_exceeded"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +@pytest.mark.parametrize("parallelism", [0, 9], ids=["below-range", "above-range"]) +def test_parallelism_outside_1_to_8_is_rejected(tmp_path: Path, parallelism: int) -> None: + manifest = _build_bundle(tmp_path) + with pytest.raises(PreflightError, match="parallelism_out_of_range"): + preflight_bundle( + tmp_path, manifest, parallelism=parallelism, secret_refs=TARGET_PROVIDER_REFS + ) + + +# --- ordering: the checklist runs in the contract's numbered order, and a bundle violating two +# rules at once must report the earlier-numbered one (F17, p4-round1-review) — every rejection +# fixture above proves presence, none of them alone proves position. -------------------------- + + +def test_a_secret_file_and_an_unknown_placeholder_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 3 (secret scan) before item 5 (placeholder vocabulary).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate, extra_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_changed_file_and_a_reserved_name_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 1 (digest verification) before item 5's reserved-name scan — the mutated file carries + both a byte-level tamper the digest catches and a reserved name the content scan would catch, + but the manifest is never resealed, so item 1 sees it first.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").write_bytes(b"CREATE TABLE _alk_conformance (id int);\n") + with pytest.raises(PreflightError, match="bundle_file_changed"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_placeholder_and_bad_parallelism_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 5 (placeholder vocabulary) before item 7 (resource sanity).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=99, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_compose_bundle_with_an_unlisted_file_reports_compose_not_hosted( + tmp_path: Path, +) -> None: + """N1 (p4-round2-review): the compose gate before item 1, before item 2 (`test_kind_compose_ + is_rejected` alone cannot prove position, since it now carries no file to trip item 2 at all). + An unlisted `compose.yaml` would report `bundle_file_unlisted` if item 2 ran first — this pins + that the gate wins instead.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + (tmp_path / "compose.yaml").write_bytes(b"services: {}\n") + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +# --- kind: external ------------------------------------------------------------------------------ + + +def test_kind_external_skips_the_process_block_but_still_enforces_resource_sanity( + tmp_path: Path, +) -> None: + """F18 (p4-round1-review): the only prior `external` coverage was model-layer + (`test_bundle_v2.py`) — nothing proved that `preflight_bundle` itself skips item 5's process + block and item 6 (`no_sql_store`) for `kind: external`, and still applies item 7.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "external"}, + "processes": [], + "seed": None, + "capabilities": { + "target": { + "protocol": "http", + "service": "customer-endpoint", + "configuration_name": "TARGET_URL", + }, + }, + }, + include_seed=False, + ) + # Item 5 (placeholder/secret-purpose/depends_on/engine-catalog/seed checks) and item 6 + # (no_sql_store) never run — nothing in this manifest could satisfy either, since it has no + # processes at all. + assert preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs={}) is None + # Item 7 still runs regardless of runtime kind. + with pytest.raises(PreflightError, match="parallelism_out_of_range"): + preflight_bundle(tmp_path, manifest, parallelism=99, secret_refs={}) From 722b12de9dd617582123486bb5b9ca3a409634d2 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 06:41:13 +0530 Subject: [PATCH 05/20] =?UTF-8?q?feat(harness):=20in-sandbox=20process=20p?= =?UTF-8?q?rovisioner=20=E2=80=94=20build,=20spawn,=20readiness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The execution half of the provisioner: per-world/job-shared port allocation, closed-vocabulary env rendering, copy-based build trees, and process spawn — every customer process dropped to its declared unprivileged user, the customer checkout copied with symlinks preserved and escaping links rejected, process names constrained so a name can never escape its build/scratch directory. depends_on waits on all of a dependency's readiness probes; healthy() only ever demotes. Managed engines start with atomically-created credential files and are probed with real queries. Signed-off-by: khushalsonawat --- src/fi/alk/harness/bundle_v2.py | 46 +- src/fi/alk/harness/process_preflight.py | 40 +- src/fi/alk/harness/process_runtime.py | 1513 +++++++++++++++++++ tests/harness/test_bundle_v2.py | 39 +- tests/harness/test_process_preflight.py | 293 +++- tests/harness/test_process_runtime.py | 1821 +++++++++++++++++++++++ 6 files changed, 3740 insertions(+), 12 deletions(-) create mode 100644 src/fi/alk/harness/process_runtime.py create mode 100644 tests/harness/test_process_runtime.py diff --git a/src/fi/alk/harness/bundle_v2.py b/src/fi/alk/harness/bundle_v2.py index 710666e0..19b228d6 100644 --- a/src/fi/alk/harness/bundle_v2.py +++ b/src/fi/alk/harness/bundle_v2.py @@ -1,5 +1,5 @@ """`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape (`hosted-execution- -seams.md` v1.7). +seams.md` v1.8). v1 (`bundle.py`) describes a `command`-per-service compose world and embeds the repository source. v2 describes `/work/source` as already present and a job that starts plain processes on @@ -29,7 +29,15 @@ from pathlib import Path from typing import Annotated, Any, Literal, Sequence, Union -from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationError, model_validator +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + ValidationError, + field_validator, + model_validator, +) from .bundle import CapabilityProtocol, _reject_secret_values, _safe_relative @@ -106,16 +114,36 @@ class SecretPurpose(str, Enum): SOURCE_CHECKOUT = "source_checkout" +# §0 (v1.8): a process `name` is path-joined into `/work/build//` and +# `/work/worlds/w//` verbatim (§2b) — the pattern below is the closed shape that makes +# `/`, `..`, and an absolute form unspellable at the model layer, matching every §2b example +# (including `tools-api`). +_PROCESS_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$") + + +def _validate_process_name(name: str) -> str: + if not _PROCESS_NAME_PATTERN.fullmatch(name): + raise ValueError(f"process_name_invalid: {name!r} must match ^[a-z0-9][a-z0-9_-]*$") + return name + + class StartedCheck(BaseModel): model_config = ConfigDict(extra="forbid") - port: int | None = Field(default=None, ge=1, le=65535) + # §2b (v1.8): "the value selects the port-probe variant, it is not a literal port number" — + # the probed port is always the dependency's own allocated port (`port_plan.port_for`, + # `process_runtime.py`), honoring `fixed_port` when the process declares one. A prior version + # of this field carried a literal int; `bool` makes the "not a literal" rule unspellable + # wrong rather than merely documented. + port: bool | None = None log_marker: str | None = None timeout_seconds: float = Field(default=30.0, gt=0) @model_validator(mode="after") def _exactly_one_probe(self) -> "StartedCheck": - if (self.port is None) == (self.log_marker is None): + has_port = bool(self.port) + has_marker = self.log_marker is not None + if has_port == has_marker: raise ValueError("started_check_requires_exactly_one_of_port_or_log_marker") return self @@ -130,6 +158,11 @@ class ManagedProcess(BaseModel): user: ProcessUser depends_on: list[str] = Field(default_factory=list) + @field_validator("name") + @classmethod + def _name_shape(cls, value: str) -> str: + return _validate_process_name(value) + class SourceProcess(BaseModel): model_config = ConfigDict(extra="forbid") @@ -147,6 +180,11 @@ class SourceProcess(BaseModel): user: ProcessUser depends_on: list[str] = Field(default_factory=list) + @field_validator("name") + @classmethod + def _name_shape(cls, value: str) -> str: + return _validate_process_name(value) + @model_validator(mode="after") def _shape(self) -> "SourceProcess": _safe_relative(self.working_directory) diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py index e774f625..1e342e4c 100644 --- a/src/fi/alk/harness/process_preflight.py +++ b/src/fi/alk/harness/process_preflight.py @@ -1,4 +1,4 @@ -"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.7 — as a single gate the +"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.8 — as a single gate the in-sandbox provisioner runs before starting anything. `bundle_v2.py` validates everything decidable from the manifest's own field values alone; this @@ -52,10 +52,14 @@ class PreflightError(RuntimeError): """A §2e checklist rule rejected the bundle. - ``code`` is one of §2e's failure-code table (v1.7): "contract-rule" codes, each named by a + ``code`` is one of §2e's failure-code table (v1.8): "contract-rule" codes, each named by a numbered checklist item's prose, and "mechanical" codes for plumbing failures the contract describes but does not formalize as a rule (a missing bundle file, an out-of-range - ``parallelism``). Every code this module raises is in that table. + ``parallelism``). Every code this module raises is in that table, with one open exception: + ``fixed_port_reserved`` (F11, p5-round1-review) has no §2e entry yet — the collision it + guards against is real (a `fixed_port` aliasing the provisioner's own port-formula bands) but + the frozen v1.8 table predates the rule; flagged for the owner to add in the next amendment, + not silently worked around. """ def __init__(self, code: str, message: str) -> None: @@ -114,6 +118,17 @@ def __init__(self, code: str, message: str) -> None: _MIN_PARALLELISM = 1 _MAX_PARALLELISM = 8 +# §2b's own port formulas (`process_runtime.plan_ports`): job-shared `14000 + ordinal` +# (ordinal <= 99, §2e item 7's process cap) and per-world `15000 + 100*world_index + ordinal` +# (world_index <= 7, §1's parallelism cap). A `fixed_port` landing inside either band can alias a +# formula port the provisioner is about to hand to a *different* process — F11, p5-round1-review. +# `fixed_port` forces W=1, so the collision surface is small, but the failure mode is a bind +# error inside a customer process, not a bundle rejection, which is strictly worse. Mirrored here +# rather than imported from `process_runtime.py`: preflight has no business depending on the +# execution module, and both bands are fixed by the contract, not by any runtime state. +_JOB_SHARED_PORT_BAND = range(14000, 14100) +_PER_WORLD_PORT_BAND = range(15000, 15800) + def preflight_bundle( bundle_dir: Path, @@ -170,6 +185,7 @@ def preflight_bundle( _verify_secret_purposes(manifest, secret_refs) # 5 _verify_depends_on(manifest) # 5 _verify_engine_catalog(manifest) # 5 + _verify_fixed_port_not_reserved(manifest) # 5 / §2b _verify_seed_missing(manifest) # 5 / §2c _verify_reserved_names(bundle_dir, manifest) # 5 _verify_seed_files_on_disk_and_listed(bundle_dir, manifest, files) # 5 @@ -469,6 +485,18 @@ def _verify_engine_catalog(manifest: EnvironmentBundleV2) -> None: ) +def _verify_fixed_port_not_reserved(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, SourceProcess) or process.fixed_port is None: + continue + if process.fixed_port in _JOB_SHARED_PORT_BAND or process.fixed_port in _PER_WORLD_PORT_BAND: + raise PreflightError( + "fixed_port_reserved", + f"{process.name}: fixed_port {process.fixed_port} falls inside the provisioner's " + "own port-formula bands (14000-14099 job-shared, 15000-15799 per-world)", + ) + + def _verify_seed_missing(manifest: EnvironmentBundleV2) -> None: covered = {store.capability for store in (manifest.seed.stores if manifest.seed else [])} missing = sorted( @@ -497,7 +525,11 @@ def _verify_reserved_names(bundle_dir: Path, manifest: EnvironmentBundleV2) -> N # a generated seed file's own note about the reservation ("-- never create # _alk_conformance here") would otherwise trip the scan on prose, not on an identifier # it defines. Quoted string literals containing the name as *data* remain a known - # false-positive surface: the scan has no lexer, only comment-stripping. + # false-positive surface: the scan has no lexer, only comment-stripping. The stripping + # is a false-NEGATIVE surface in the opposite direction, equally lexer-free and equally + # left as-is (N7, p4-round2-review; B4, p4-round3-review): a `--` or `/*` inside a + # string literal (not a comment) deletes real content up to the next line-end or `*/`, + # which can delete a reserved-name definition that follows it on the same statement. code = _SQL_BLOCK_COMMENT.sub("", _SQL_LINE_COMMENT.sub("", text)) if _RESERVED_NAME_PATTERN.search(code): raise PreflightError( diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py new file mode 100644 index 00000000..ddaafbad --- /dev/null +++ b/src/fi/alk/harness/process_runtime.py @@ -0,0 +1,1513 @@ +"""The execution half of the in-sandbox provisioner — `hosted-execution-seams.md` v1.8, §2b/§3/§4. + +Builds one world's running processes from an already-`preflight_bundle`-cleared +`EnvironmentBundleV2`: port allocation, `{{...}}` placeholder rendering, copy-based build trees, +process spawn (managed engines and `source` processes), `depends_on` wait, and `healthy()` +readiness probing. This is `provision()` UP TO process spawn only — seed/baseline application, +the world-reset mechanism, and the conformance gate (§4) are a later phase, deliberately left out +so they compose on top of `spawn_world`/`build_process_tree` rather than reworking them. The +`RuntimeProvider` Protocol itself (`runtime.py`) is untouched here; a later phase wires a +Protocol-conforming, stateful adapter around the pure functions below. + +Nothing here assumes Docker, a container runtime, or a network provider — §0: "No Docker inside +the sandbox." Every engine/process concept is a plain `subprocess`, matched against the module's +own `ProcessRunner`/`CapabilityProber` seams so tests substitute fakes; the one exception +(`default_capability_prober`'s postgres branch) import-guards `psycopg` and falls back to a bare +TCP probe when it is absent, the same fallback a bundle's own store would get before a role or +database exists to authenticate against. + +§0/§2b `user`: every build tree and every spawned process is chowned/spawned under its declared +`user`, resolved through `pwd.getpwnam` (`default_user_resolver`). A dev box has none of the +snapshot's `svc-*` accounts, so resolution failing is the expected local-lane shape, not an error +by itself — `require_declared_user=False` (the default) falls back to running unprivileged and +logs it; a hosted caller that wants a missing user to be a typed failure instead sets it `True`. +""" + +from __future__ import annotations + +import logging +import os +import pwd +import re +import secrets +import shutil +import socket +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Callable, Protocol, Sequence +from urllib.parse import urlsplit + +from pydantic import BaseModel, Field, JsonValue + +from .bundle import CapabilityProtocol +from .bundle_v2 import ( + BaselineStrategy, + CapabilityV2, + EnvironmentBundleV2, + ManagedEngine, + ManagedProcess, + ProcessUser, + ReadinessProbeV2, + SecretPurpose, + SourceProcess, +) + +logger = logging.getLogger(__name__) + +# --- errors -------------------------------------------------------------------------------- + + +class ProcessRuntimeError(RuntimeError): + """A runtime-execution failure below the §2e preflight gate. + + The bundle has already passed `preflight_bundle` by the time any of this module runs, so + these are process/filesystem/timing failures. v1.8 added §2f: a CLOSED table for the subset + of these that cross the outbound seam (`source_tree_unavailable`, `build_failed`, + `runtime_unsupported`, `spawn_failed`, `depends_on_timeout`, `unsupported_capability_protocol`), + each mapped to a `FailureDomain` per §4.6 — every code this module raises THAT crosses the + seam is in that table. A handful of `code` values prefixed `internal_` + (`internal_unknown_placeholder`, `internal_missing_credentials`) are deliberately NOT in it: + each marks a precondition `preflight_bundle` should already have made impossible, so it is a + bug to fix here, not a failure the outbound seam ever needs a name for. `stage` names which + phase failed (`build`, `spawn`, `depends_on`, `render`); `process` names the process involved, + when there is one. + """ + + def __init__(self, stage: str, code: str, message: str, *, process: str | None = None) -> None: + self.stage = stage + self.code = code + self.process = process + located = f" ({process})" if process else "" + super().__init__(f"{stage}/{code}{located}: {message}") + + +# --- §3 EnvironmentRuntime ------------------------------------------------------------------- + + +class RuntimeState(str, Enum): + PREPARING = "preparing" + READY = "ready" + UNHEALTHY = "unhealthy" + STOPPED = "stopped" + + +class RuntimeEndpoint(BaseModel): + capability: str + protocol: str + address: str + configuration_name: str | None = None + + +class EnvironmentRuntime(BaseModel): + """§3's per-world provisioner output. No `provider` field (§4 delta: "no remote provider + exists at this seam") — the class attribute `RuntimeProvider.name` (`runtime.py`) is retained + there for logging only, not carried onto each world's own record.""" + + runtime_id: str + world_index: int = Field(ge=0) + bundle_digest: str + state: RuntimeState + endpoints: dict[str, RuntimeEndpoint] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) + + +def new_runtime_id(bundle_digest: str, world_index: int) -> str: + """Opaque per §3 ("nothing parses it") — still traceable in logs without decoding.""" + return f"{bundle_digest}:w{world_index}:{secrets.token_hex(4)}" + + +# --- §2b port allocation --------------------------------------------------------------------- + +_PER_WORLD_BASE = 15000 +_PER_WORLD_STRIDE = 100 +_JOB_SHARED_BASE = 14000 + + +def _ordinal_map(manifest: EnvironmentBundleV2) -> dict[str, int]: + """§2b: "ordinal = the process's 0-based index in the `processes` array as authored" — one + map, shared by both port ranges.""" + return {process.name: index for index, process in enumerate(manifest.processes)} + + +def _job_shared_process_names(manifest: EnvironmentBundleV2) -> frozenset[str]: + """§2b instancing rule: a managed engine runs once per job iff some store backed by one of + its capabilities uses `template_database`; `datadir_copy`, `empty`, or no `seed.stores` entry + at all is per-world — per-world is the only safe default for an engine one world's reset + could otherwise corrupt for the others. Only postgres's catalog entry permits + `template_database` at all (`bundle_v2._ENGINE_STRATEGIES`), so this can never mark a redis + or rabbitmq process job-shared. + """ + service_by_capability = {slug: cap.service for slug, cap in manifest.capabilities.items()} + strategies_by_service: dict[str, set[BaselineStrategy]] = {} + if manifest.seed is not None: + for store in manifest.seed.stores: + service = service_by_capability.get(store.capability) + if service is not None: + strategies_by_service.setdefault(service, set()).add(store.baseline.strategy) + return frozenset( + process.name + for process in manifest.processes + if isinstance(process, ManagedProcess) + and BaselineStrategy.TEMPLATE_DATABASE in strategies_by_service.get(process.name, set()) + ) + + +@dataclass(frozen=True) +class PortPlan: + """A job's whole port assignment, computed once from the manifest and the requested instance + count. `fixed_port` (§2b) forces `effective_instances` to 1 — `port_for` honors the fixed + value exactly for that process, in every world index, which is consistent because there can + only ever be world 0 once `effective_instances` is 1. Emitting the `parallelism_degraded` + event for `degraded_reason` is the scheduler's job (§2b), not this module's — this only + records the fact. + """ + + ordinals: dict[str, int] + job_shared: frozenset[str] + fixed_ports: dict[str, int] + effective_instances: int + degraded_reason: str | None + + def is_job_shared(self, process_name: str) -> bool: + return process_name in self.job_shared + + def port_for(self, process_name: str, world_index: int) -> int: + if process_name in self.fixed_ports: + return self.fixed_ports[process_name] + ordinal = self.ordinals[process_name] + if process_name in self.job_shared: + return _JOB_SHARED_BASE + ordinal + return _PER_WORLD_BASE + _PER_WORLD_STRIDE * world_index + ordinal + + +def plan_ports(manifest: EnvironmentBundleV2, *, instances: int) -> PortPlan: + ordinals = _ordinal_map(manifest) + job_shared = _job_shared_process_names(manifest) + fixed_ports = { + process.name: process.fixed_port + for process in manifest.processes + if isinstance(process, SourceProcess) and process.fixed_port is not None + } + return PortPlan( + ordinals=ordinals, + job_shared=job_shared, + fixed_ports=fixed_ports, + effective_instances=1 if fixed_ports else instances, + degraded_reason="fixed_port" if fixed_ports else None, + ) + + +# --- engine credentials, generated once per job ----------------------------------------------- + + +@dataclass(frozen=True) +class EngineCredentials: + username: str + password: str + + +def generate_engine_credentials( + manifest: EnvironmentBundleV2, *, token: Callable[[], str] | None = None +) -> dict[str, EngineCredentials]: + """§2b catalog: postgres/rabbitmq use role/user `harness`, "password generated per job"; + redis carries no auth in V1 and gets no entry here. Keyed by process name (not engine) since + a job can carry more than one postgres process. `token` is injectable for deterministic + tests; production callers rely on the `secrets` module default. + """ + make_token = token or (lambda: secrets.token_urlsafe(24)) + return { + process.name: EngineCredentials(username="harness", password=make_token()) + for process in manifest.processes + if isinstance(process, ManagedProcess) + and process.engine in (ManagedEngine.POSTGRES, ManagedEngine.RABBITMQ) + } + + +def render_capability_address( + capability: CapabilityV2, + *, + port: int, + world_index: int, + credentials: EngineCredentials | None, +) -> str: + """§2b: `{{DATABASE_URL}}` etc. render "with the catalog role, the generated password, the + allocated port, and `{{DB_NAME}}`." `{{DB_NAME}}` is always `w` (§2b), including under + `template_database`, where the logical database differs per world on one shared server.""" + host = "localhost" + protocol = capability.protocol + if protocol is CapabilityProtocol.POSTGRES: + if credentials is None: + # S11/P4's N8(2) precedent: a bare `assert` for a real precondition is stripped under + # `python -O` — this is a should-never-happen internal invariant (`generate_engine_ + # credentials` always produces one for every postgres/rabbitmq `ManagedProcess`), not + # a bundle defect, so it is `internal_`-prefixed, not a §2f code. + raise ProcessRuntimeError( + "render", "internal_missing_credentials", + "postgres capability requires generated credentials but none were supplied", + ) + auth = f"{credentials.username}:{credentials.password}" + return f"postgresql://{auth}@{host}:{port}/w{world_index}" + if protocol is CapabilityProtocol.AMQP: + if credentials is None: + raise ProcessRuntimeError( + "render", "internal_missing_credentials", + "amqp capability requires generated credentials but none were supplied", + ) + return f"amqp://{credentials.username}:{credentials.password}@{host}:{port}/" + if protocol is CapabilityProtocol.REDIS: + return f"redis://{host}:{port}" + if protocol is CapabilityProtocol.HTTP: + return f"http://{host}:{port}" + # F10, p5-round1-review: no other capability protocol (grpc, mongodb, s3, ...) has a defined + # address shape at this seam — §3 names exactly two worked examples. A bare + # `://host:port` used to be handed to a customer process as its own + # `{{CONFIGURATION_NAME}}` value, which is not a working address for any of those protocols; + # failing loudly beats handing out an address that cannot work. + raise ProcessRuntimeError( + "render", "unsupported_capability_protocol", + f"{protocol.value} has no defined address shape at this seam", + ) + + +def build_endpoints( + manifest: EnvironmentBundleV2, + *, + world_index: int, + port_plan: PortPlan, + credentials: dict[str, EngineCredentials], +) -> dict[str, RuntimeEndpoint]: + """§3's `endpoints` map, built fresh for one world — every declared capability gets an + entry regardless of whether any process placeholder ever references its `configuration_name` + (a null `configuration_name` is legal; the scheduler/simulator still read endpoints by + capability slug, per §3's own reader list).""" + endpoints: dict[str, RuntimeEndpoint] = {} + for slug, capability in manifest.capabilities.items(): + port = port_plan.port_for(capability.service, world_index) + address = render_capability_address( + capability, + port=port, + world_index=world_index, + credentials=credentials.get(capability.service), + ) + endpoints[slug] = RuntimeEndpoint( + capability=slug, + protocol=capability.protocol.value, + address=address, + configuration_name=capability.configuration_name, + ) + return endpoints + + +def configuration_addresses_from_endpoints( + endpoints: dict[str, RuntimeEndpoint], +) -> dict[str, str]: + return { + endpoint.configuration_name: endpoint.address + for endpoint in endpoints.values() + if endpoint.configuration_name + } + + +# --- §2b placeholder renderer ------------------------------------------------------------------ + +_PLACEHOLDER = re.compile(r"\{\{([^{}]+)\}\}") +_NAMED_PLACEHOLDER = re.compile(r"^(PORT|HOST)_(.+)$") + + +def render_template( + value: str, + *, + process_name: str, + world_index: int, + world_dir: Path, + port_plan: PortPlan, + configuration_addresses: dict[str, str], +) -> str: + """§2b's closed placeholder vocabulary. `preflight_bundle` has already validated every token + in `value` against this exact vocabulary before this ever runs, so a token this function + cannot resolve is an internal bug, not a bundle defect — raised as `ProcessRuntimeError`, + never one of `PreflightError`'s §2e codes (those are preflight's alone; see its own + docstring: "crosses the outbound seam").""" + + def resolve(match: re.Match[str]) -> str: + token = match.group(1) + if token == "WORLD_INDEX": + return str(world_index) + if token == "WORLD_DIR": + return str(world_dir) + if token == "DB_NAME": + return f"w{world_index}" + named = _NAMED_PLACEHOLDER.match(token) + if named: + kind, name = named.groups() + if kind == "HOST": + return "localhost" + try: + return str(port_plan.port_for(name, world_index)) + except KeyError: + raise ProcessRuntimeError( + "render", + "internal_unknown_placeholder", + f"{{{{{token}}}}} names {name!r}, which is not in this bundle's processes", + process=process_name, + ) from None + if token in configuration_addresses: + return configuration_addresses[token] + raise ProcessRuntimeError( + "render", + "internal_unknown_placeholder", + f"{{{{{token}}}}} has no resolution; preflight should have rejected this bundle", + process=process_name, + ) + + return _PLACEHOLDER.sub(resolve, value) + + +def render_environment( + process: SourceProcess, + *, + world_index: int, + world_dir: Path, + port_plan: PortPlan, + configuration_addresses: dict[str, str], +) -> dict[str, str]: + """`build_environment` is deliberately excluded — §2b: it "takes NO placeholders at all," so + it is merged raw by the env builders below, never passed through this renderer.""" + return { + key: render_template( + value, + process_name=process.name, + world_index=world_index, + world_dir=world_dir, + port_plan=port_plan, + configuration_addresses=configuration_addresses, + ) + for key, value in process.environment.items() + } + + +def select_process_secrets( + process: SourceProcess, *, secret_values: dict[str, str], secret_purposes: dict[str, str] +) -> dict[str, str]: + """§2b: "the provisioner injects every alias whose ref's `purpose` is listed [in + `secret_purposes`], under the **alias** as the env-var name." `secret_purposes` maps each + job-level alias to its `SecretRef.purpose` value — the same shape `process_preflight. + preflight_bundle`'s own `secret_refs` argument uses, so a caller that already ran preflight + has this for free. + + `SecretPurpose.SOURCE_CHECKOUT` is excluded unconditionally (F13, p5-round1-review): §1 states + it is "gateway-only; never uploaded to the guest," and preflight does not forbid a process + from legally *claiming* it (§2b's `secret_unclaimed`/`secret_missing` pair is scoped to + `target_provider` only) — the guest should not depend on the gateway alone never putting one + in `secrets.json` to keep that promise. + """ + claimed = { + purpose.value + for purpose in process.secret_purposes + if purpose is not SecretPurpose.SOURCE_CHECKOUT + } + return { + alias: value + for alias, value in secret_values.items() + if secret_purposes.get(alias) in claimed + } + + +# §2b enumerates exactly what a process receives: rendered `environment`, `build_environment`, +# purpose-matched secrets, and the PATH prepend below — the ambient `svc-control` environment is +# not on that list, and §1 marks the source untrusted (F12, p5-round1-review). A short, fixed +# allowlist of the interpreter/locale plumbing a process cannot run without, rather than +# `os.environ` wholesale — nothing here exports a secret today, but the entrypoint that will +# (a bearer token, a `FUTUREAGI_*` marker) is exactly the next thing wired on top of this module. +_INHERITED_ENV_ALLOWLIST = ("PATH", "HOME", "LANG", "TZ", "TMPDIR") + + +def _allowlisted_ambient_env(source: dict[str, str]) -> dict[str, str]: + env = {key: value for key, value in source.items() if key in _INHERITED_ENV_ALLOWLIST} + env.update({key: value for key, value in source.items() if key.startswith("LC_")}) + return env + + +def _base_process_env( + build_dir: Path, extra: dict[str, str] | None = None, *, base: dict[str, str] | None = None +) -> dict[str, str]: + """§2b: `build_environment` "merged" env, plus the provisioner's own unconditional PATH + prepend — applied last so it always wins even if `extra` (or the ambient environment) sets + its own `PATH`, for both build and run per the contract's own words. `base` defaults to an + allowlisted slice of the ambient environment (F12), not `os.environ` wholesale.""" + env = dict(base if base is not None else _allowlisted_ambient_env(os.environ)) + if extra: + env.update(extra) + prepend = [str(build_dir / ".venv" / "bin"), str(build_dir / "node_modules" / ".bin")] + # F14, p5-round1-review: `filter(None, ...)` — an unset/empty `PATH` would otherwise leave a + # trailing `:`, and POSIX `execvp` reads an empty PATH element as "current directory." cwd for + # both build and run is the customer's own build tree, so that would let a repo shipping an + # executable literally named `ls`/`git`/`sh` shadow the real one for any bare-name argv[0]. + env["PATH"] = os.pathsep.join(filter(None, [*prepend, env.get("PATH", "")])) + return env + + +# --- §2b copy-based build trees ----------------------------------------------------------------- + +# §0 (v1.7): "a repo needing an interpreter the snapshot lacks fails at BUILD time... reported +# `runtime_unsupported`, naming what the snapshot ships." Detection surface: an exec of a build +# step's argv[0] that looks like an interpreter name (`python`, `python3`, `python3.11`, `node`, +# `node20`, ...) raising `FileNotFoundError` — anything else that fails to exec, or exits +# nonzero, is `build_failed`. This is a judgement call the contract states as a rule +# ("the build step's failure is reported `runtime_unsupported`") without naming a detection +# mechanism; §0 also promises "python 3.11 and 3.12, node 20 and 22" specifically, so the pattern +# is scoped to those two families rather than every possible interpreter name. +_INTERPRETER_PATTERN = re.compile(r"^(python\d*(\.\d+)?|node\d*)(\.exe)?$") + + +def _looks_like_missing_interpreter(argv0: str) -> bool: + return bool(_INTERPRETER_PATTERN.match(Path(argv0).name)) + + +# --- process identity: user resolution and path containment (F1/F3, p5-round1-review) --------- + + +def default_user_resolver(username: str) -> "pwd.struct_passwd | None": + """§0: the base snapshot guarantees `svc-agent`/`svc-tools`/`svc-data` exist — on the hosted + path this always resolves. A dev box has none of them; returning `None` rather than raising + is what lets `_resolve_process_user`'s local-lane fallback work without a second seam.""" + try: + return pwd.getpwnam(username) + except KeyError: + return None + + +def _resolve_process_user( + user: ProcessUser, + *, + resolver: Callable[[str], "pwd.struct_passwd | None"], + require: bool, + process_name: str, + stage: str, +) -> "pwd.struct_passwd | None": + """F1, p5-round1-review: every build tree and every spawned process must run under its + declared `user`, not the harness's own `svc-control` — otherwise an untrusted `agent` process + runs with read access to the whole-attempt capabilities bearer (§0 step 4) and, since every + process then shares one uid, mutual `/proc//environ` visibility into every other + process's injected secrets regardless of `secret_purposes`. + + `require=False` (the default, the local test lane's shape — no `svc-*` accounts on a dev box) + logs and returns `None`, so the caller runs unprivileged rather than failing every local run. + `require=True` is for a caller that knows it is on the hosted path, where the snapshot's own + guarantee means resolution failing is itself an infrastructure fault worth a typed failure. + """ + resolved = resolver(user.value) + if resolved is None: + if require: + raise ProcessRuntimeError( + stage, "spawn_failed", + f"{user.value!r} has no passwd entry; the hosted snapshot must guarantee it", + process=process_name, + ) + logger.warning( + "process %s declares user=%s but it is not resolvable on this host; running " + "unprivileged (local test lane fallback, not the hosted path)", + process_name, user.value, + ) + return resolved + + +def _default_chown(path: Path, uid: int, gid: int) -> None: + # `follow_symlinks=False`: F2 copies a within-tree symlink as a link, never dereferenced — + # chowning through it would touch whatever it points at instead of the link itself. + os.chown(path, uid, gid, follow_symlinks=False) + + +def _chown_tree(root: Path, *, uid: int, gid: int, chown: Callable[[Path, int, int], None]) -> None: + chown(root, uid, gid) + for dirpath, dirnames, filenames in os.walk(root): + for name in (*dirnames, *filenames): + chown(Path(dirpath) / name, uid, gid) + + +def _ensure_within(path: Path, root: Path, *, process_name: str, stage: str) -> Path: + """Defense in depth for F3, p5-round1-review. The model layer now constrains a process `name` + to `^[a-z0-9][a-z0-9_-]*$` (`bundle_v2.py`), which already makes a `/`, `..`, or absolute name + unspellable — a process named `/etc` used to path-join into `build_dir = Path("/etc")` + directly, which the very next line then `rmtree`'d as `svc-control`. This is the backstop for + anything that reaches a name-derived directory without going through that validator, checked + right before it is handed to `rmtree`/`mkdir`. Returns the ORIGINAL `path`, not + `path.resolve()`'d — this is a containment check, not a normalization; returning the resolved + form would silently rewrite every caller's path (e.g. through a `/tmp` -> `/private/tmp` + symlink on macOS) for no reason connected to the check itself. + """ + resolved = path.resolve() + resolved_root = root.resolve() + if not resolved.is_relative_to(resolved_root): + raise ProcessRuntimeError( + stage, "process_name_invalid", + f"{process_name!r} resolves to {resolved}, which escapes {resolved_root}", + process=process_name, + ) + return path + + +def _reject_escaping_symlinks(tree_root: Path, allowed_root: Path, *, process_name: str) -> None: + """F2, p5-round1-review: §2e item 2's symlink rejection is scoped to the bundle's OWN files — + nothing scans `/work/source` for a symlink pointing outside it, since the bundle "does NOT + embed the repository source" (§2 preamble). Left unchecked, a repo can ship + `config/creds.json -> /run/futureagi/capabilities.json` and have the provisioner (running as + svc-control until F1's privilege drop even applies) materialize that 0600 file's bytes into + the customer's own build tree. Walked against the SOURCE tree before anything is copied, so a + rejection never partially materializes a tree first. `.resolve()` (not raw `os.readlink`) + deliberately, so a relative target or a multi-hop symlink chain is followed all the way to + where it actually lands, not just its first hop. + """ + for entry in tree_root.rglob("*"): + if not entry.is_symlink(): + continue + target = entry.resolve() + if not target.is_relative_to(allowed_root): + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{entry.relative_to(tree_root)} is a symlink to {target}, which escapes " + "/work/source", + process=process_name, + ) + + +def _copytree_preserving_symlinks(src: Path, dst: Path) -> None: + # F2: `symlinks=True` — copy a link as a link rather than dereferencing it. `shutil. + # copytree`'s default (`symlinks=False`) follows every symlink and copies the TARGET's + # contents as a regular file, which is what let a symlink out of the checkout read as + # svc-control in the first place. A link that escaped `/work/source` was already rejected by + # `_reject_escaping_symlinks` before this ever runs; one that stays inside the tree is copied + # as-is and simply works (or dangles harmlessly) under the chowned, unprivileged build tree. + shutil.copytree(src, dst, symlinks=True) + + +_DEFAULT_BUILD_STEP_TIMEOUT_SECONDS = 600.0 + + +def build_process_tree( + process: SourceProcess, + *, + source_root: Path, + build_root: Path, + run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + copy: Callable[[Path, Path], None] | None = None, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS, +) -> Path: + """§2b: copy `source_root/` to `build_root//`, chowned to the + process's `user` (F1), then argv-exec each `build_commands` step there under that same user — + no shell, so `&&`/`$VAR`/globs/pipes do not work, which is why the model layer + (`bundle_v2.SourceProcess._shape`) already rejects an empty step. Runs once; the caller is + responsible for calling this exactly once per job, per process — this function itself has no + per-job memory. Raises on the first failing step (`ProcessRuntimeError`, stage="build"); never + partially succeeds silently past a failure. + """ + build_dir = build_root / process.name + _ensure_within(build_dir, build_root, process_name=process.name, stage="build") + + source_dir = source_root / process.working_directory + resolved_source_root = source_root.resolve() + resolved_source_dir = source_dir.resolve() + if not resolved_source_dir.is_relative_to(resolved_source_root): + # F2: a symlinked PATH COMPONENT (e.g. `/work/source/services` itself a symlink to `/`) + # used to be silently followed by the old bare `.resolve()` — `working_directory` passes + # the model layer's `_safe_relative` (no `..`, not absolute) without ever naming the + # escape. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{process.working_directory} resolves outside /work/source", + process=process.name, + ) + if not resolved_source_dir.is_dir(): + # F5, p5-round1-review: named explicitly, before any copy attempt, rather than letting + # `copytree`'s own `FileNotFoundError`/`NotADirectoryError` climb out untyped. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{process.working_directory} is absent or not a directory in the checkout", + process=process.name, + ) + _reject_escaping_symlinks(resolved_source_dir, resolved_source_root, process_name=process.name) + + if build_dir.exists(): + shutil.rmtree(build_dir) + try: + (copy or _copytree_preserving_symlinks)(resolved_source_dir, build_dir) + except (OSError, shutil.Error) as exc: + # F5: `PermissionError` on an unreadable subtree, `shutil.Error`'s aggregated failure on a + # dangling symlink — every copy-phase failure is the same deterministic, non-retryable + # fault as a missing directory, so it gets the same code. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"copying {process.working_directory}: {exc}", process=process.name, + ) from exc + + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="build", + ) + if resolved_user is not None: + _chown_tree(build_dir, uid=resolved_user.pw_uid, gid=resolved_user.pw_gid, chown=chown) + spawn_uid = resolved_user.pw_uid if resolved_user is not None else None + spawn_gid = resolved_user.pw_gid if resolved_user is not None else None + + env = _base_process_env(build_dir, process.build_environment) + for step in process.build_commands: + try: + result = run( + step, cwd=build_dir, env=env, capture_output=True, text=True, + timeout=build_step_timeout_seconds, user=spawn_uid, group=spawn_gid, + ) + except subprocess.TimeoutExpired as exc: + # F15, p5-round1-review: an install step wedged on a private registry with no DNS + # answer used to block the provisioner forever — the only backstop was the gateway's + # whole-job TTL, which arrives as SIGTERM to the entrypoint while this call is still + # inside an uninterruptible `subprocess.run`. + raise ProcessRuntimeError( + "build", "build_failed", + f"{step!r} exceeded the {build_step_timeout_seconds}s build-step timeout", + process=process.name, + ) from exc + except FileNotFoundError as exc: + if not build_dir.is_dir(): + # JC1 ruling (p5-round1-review): a vanished build tree plus a `python*`/`node*` + # step raises the exact same `FileNotFoundError` as a missing interpreter — this + # disambiguates a filesystem fault from an interpreter-availability one before the + # argv[0] heuristic below ever gets a say. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{build_dir} vanished before {step!r} could run", process=process.name, + ) from exc + if _looks_like_missing_interpreter(step[0]): + raise ProcessRuntimeError( + "build", + "runtime_unsupported", + f"{step[0]!r} is not on the snapshot's PATH; the snapshot ships python " + "3.11/3.12 and node 20/22 only", + process=process.name, + ) from exc + raise ProcessRuntimeError( + "build", "build_failed", f"{step!r}: {exc}", process=process.name + ) from exc + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "build", + "build_failed", + f"{step!r} exited {result.returncode}" + (f": {stderr}" if stderr else ""), + process=process.name, + ) + return build_dir + + +# --- process spawn ------------------------------------------------------------------------------- + + +class SpawnedProcess(Protocol): + """What both a real `subprocess.Popen` wrapper and a test fake must provide — enough for + `depends_on`'s `log_marker` variant and for cleanup, nothing engine-specific.""" + + def is_running(self) -> bool: ... + + def captured_output(self) -> str: ... + + def terminate(self) -> None: ... + + +class ProcessRunner(Protocol): + def __call__( + self, argv: Sequence[str], *, cwd: Path, env: dict[str, str], log_path: Path, + user: int | None = None, group: int | None = None, + ) -> SpawnedProcess: ... + + +class CapabilityProber(Protocol): + def __call__( + self, *, protocol: CapabilityProtocol, host: str, port: int, path: str | None, + user: str | None = None, password: str | None = None, dbname: str | None = None, + ) -> bool: ... + + +@dataclass +class PopenProcess: + """`SpawnedProcess` backed by a real subprocess. stdout+stderr are redirected to a log file + under the process's own scratch directory rather than captured through a pipe-reading thread + — simpler, and `started_check`'s `log_marker` only ever needs to re-read the file's current + contents, not race a live stream.""" + + popen: subprocess.Popen + log_path: Path + + def is_running(self) -> bool: + return self.popen.poll() is None + + def captured_output(self) -> str: + try: + return self.log_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + def terminate(self) -> None: + self.popen.terminate() + + +def _default_log_chown(path: Path, uid: int, gid: int) -> None: + os.chown(path, uid, gid) + + +def default_process_runner( + argv: Sequence[str], *, cwd: Path, env: dict[str, str], log_path: Path, + user: int | None = None, group: int | None = None, + chown: Callable[[Path, int, int], None] = _default_log_chown, +) -> PopenProcess: + log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = log_path.open("wb") + if user is not None: + # F1, p5-round1-review: the log file is created by the harness (svc-control) before the + # child's privilege drop — the child can still WRITE through the inherited fd regardless + # of on-disk mode (permission checks happen at open(), not at write()), but leaving it + # svc-control-owned means nothing running as the child's own user could ever open it + # fresh afterward. Chowned to match, not just handed over unchecked. `chown` is injectable + # like every other chown call in this module (`os.chown` to a foreign uid needs root/ + # CAP_CHOWN — the same privilege `user=`/`group=` below already requires, so a real + # hosted deployment has both or neither; a dev-box test fakes it structurally). + chown(log_path, user, group if group is not None else -1) + popen = subprocess.Popen( + list(argv), cwd=cwd, env=env, stdout=log_file, stderr=subprocess.STDOUT, + user=user, group=group, + ) + return PopenProcess(popen=popen, log_path=log_path) + + +@dataclass +class SpawnedWorldProcess: + process_name: str + handle: SpawnedProcess + port: int + world_index: int | None # None for a job-shared managed engine + + +# --- managed-engine launch commands --------------------------------------------------------- +# +# §2b fixes the catalog (engine/version/role/db-name/strategies) and the port formula; it does +# not fix an exact launch invocation for any of the three engines — that is provisioner-internal, +# same status as `LocalComposeRuntimeProvider`'s own Compose mechanics. The commands below are a +# defensible V1 default, kept swappable through `ProcessRunner`/`sync_run` injection; they are +# never executed in this module's own test lane (§0: assumed on PATH, never required in tests). + + +def postgres_bootstrap_argv( + *, data_dir: Path, credentials: EngineCredentials, pwfile: Path +) -> list[str]: + return [ + "initdb", "-D", str(data_dir), "-U", credentials.username, + "--pwfile", str(pwfile), "-A", "scram-sha-256", + ] + + +def postgres_daemon_argv(*, data_dir: Path, port: int) -> list[str]: + return [ + "postgres", "-D", str(data_dir), "-p", str(port), "-k", str(data_dir), "-h", "localhost", + ] + + +def redis_daemon_argv(*, data_dir: Path, port: int) -> list[str]: + return [ + "redis-server", "--port", str(port), "--dir", str(data_dir), + "--daemonize", "no", "--save", "", + ] + + +def rabbitmq_daemon_argv() -> list[str]: + return ["rabbitmq-server"] + + +def rabbitmq_daemon_env( + *, data_dir: Path, port: int, credentials: EngineCredentials +) -> dict[str, str]: + return { + "RABBITMQ_NODE_PORT": str(port), + "RABBITMQ_MNESIA_BASE": str(data_dir), + "RABBITMQ_LOG_BASE": str(data_dir), + "RABBITMQ_DEFAULT_USER": credentials.username, + "RABBITMQ_DEFAULT_PASS": credentials.password, + "RABBITMQ_NODENAME": f"harness-{port}@localhost", + } + + +def spawn_managed_process( + process: ManagedProcess, + *, + port: int, + data_dir: Path, + credentials: EngineCredentials | None, + runner: ProcessRunner = default_process_runner, + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, +) -> SpawnedWorldProcess: + """Spawns one managed-engine daemon. Postgres alone needs a one-time synchronous bootstrap + (`initdb`) before the daemon can start at all — run through `sync_run`, not `runner`, since + it must complete before the daemon exec happens, not run alongside it. Redis/RabbitMQ + initialize their own data directory on first boot and need no separate step. + + `data_dir` is chowned to the engine's declared `user` (`svc-data`, per §2b) before anything + runs in it (F1, p5-round1-review) — otherwise a daemon spawned under that user could not even + write its own data directory, which the harness (running as `svc-control`) created. + """ + data_dir.mkdir(parents=True, exist_ok=True) + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="spawn", + ) + if resolved_user is not None: + chown(data_dir, resolved_user.pw_uid, resolved_user.pw_gid) + spawn_uid = resolved_user.pw_uid if resolved_user is not None else None + spawn_gid = resolved_user.pw_gid if resolved_user is not None else None + + env = _allowlisted_ambient_env(os.environ) + if process.engine is ManagedEngine.POSTGRES: + if credentials is None: + raise ProcessRuntimeError( + "spawn", "spawn_failed", "postgres requires generated credentials", + process=process.name, + ) + if not (data_dir / "PG_VERSION").exists(): + pwfile = data_dir.parent / f".{process.name}.pwfile" + # F7, p5-round1-review: created at 0600 ATOMICALLY. `write_text` then `chmod` creates + # the file at `0666 & ~umask` (typically 0644) and only narrows it afterward — a + # classic create-then-chmod TOCTOU that leaves the generated superuser password + # world-readable for the window in between. `O_EXCL` additionally refuses to follow a + # pre-planted symlink/file already sitting at this path. + fd = os.open(pwfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(credentials.password + "\n") + if resolved_user is not None: + # `initdb` itself runs as `resolved_user` below (via `sync_run`'s `user=`) — + # it must be able to READ its own 0600 file, so ownership follows the same + # user, never the mode widening to let anyone else read it too. + chown(pwfile, resolved_user.pw_uid, resolved_user.pw_gid) + bootstrap_argv = postgres_bootstrap_argv( + data_dir=data_dir, credentials=credentials, pwfile=pwfile + ) + result = sync_run( + bootstrap_argv, capture_output=True, text=True, + user=spawn_uid, group=spawn_gid, + ) + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "spawn", "spawn_failed", + f"initdb exited {result.returncode}: {stderr}", + process=process.name, + ) + finally: + pwfile.unlink(missing_ok=True) + argv = postgres_daemon_argv(data_dir=data_dir, port=port) + elif process.engine is ManagedEngine.REDIS: + argv = redis_daemon_argv(data_dir=data_dir, port=port) + elif process.engine is ManagedEngine.RABBITMQ: + if credentials is None: + raise ProcessRuntimeError( + "spawn", "spawn_failed", "rabbitmq requires generated credentials", + process=process.name, + ) + env.update(rabbitmq_daemon_env(data_dir=data_dir, port=port, credentials=credentials)) + argv = rabbitmq_daemon_argv() + else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"unknown engine {process.engine!r}", process=process.name + ) + try: + handle = runner( + argv, cwd=data_dir, env=env, log_path=data_dir / "process.log", + user=spawn_uid, group=spawn_gid, + ) + except FileNotFoundError as exc: + raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc + return SpawnedWorldProcess( + process_name=process.name, handle=handle, port=port, world_index=None + ) + + +def spawn_source_process( + process: SourceProcess, + *, + build_dir: Path, + world_dir: Path, + world_index: int, + port_plan: PortPlan, + configuration_addresses: dict[str, str], + secret_values: dict[str, str], + secret_purposes: dict[str, str], + runner: ProcessRunner = default_process_runner, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, +) -> SpawnedWorldProcess: + """§2b: `run_command` "exec'd once per world with cwd `/work/build//`" — the build tree + is read-only by convention at run time; `world_dir` (`{{WORLD_DIR}}`) is this process's own + per-world writable scratch, created here since nothing upstream of spawn needs it to exist. + Chowned to the process's declared `user` before spawn (F1, p5-round1-review) — otherwise a + process running as anyone but the harness's own uid could not write into its own `{{WORLD_ + DIR}}` at all, since the harness (as `svc-control`) is the one that just created it. + """ + world_dir.mkdir(parents=True, exist_ok=True) + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="spawn", + ) + if resolved_user is not None: + chown(world_dir, resolved_user.pw_uid, resolved_user.pw_gid) + rendered = render_environment( + process, + world_index=world_index, + world_dir=world_dir, + port_plan=port_plan, + configuration_addresses=configuration_addresses, + ) + injected = select_process_secrets( + process, secret_values=secret_values, secret_purposes=secret_purposes + ) + env = _base_process_env( + build_dir, {**(process.build_environment or {}), **rendered, **injected} + ) + try: + handle = runner( + list(process.run_command), cwd=build_dir, env=env, log_path=world_dir / "process.log", + user=resolved_user.pw_uid if resolved_user is not None else None, + group=resolved_user.pw_gid if resolved_user is not None else None, + ) + except FileNotFoundError as exc: + raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc + port = port_plan.port_for(process.name, world_index) + return SpawnedWorldProcess( + process_name=process.name, handle=handle, port=port, world_index=world_index + ) + + +# --- filesystem layout (§0's guaranteed paths, plus one this module must invent) -------------- + + +def build_tree_dir(work_directory: Path, process_name: str) -> Path: + return _ensure_within( + work_directory / "build" / process_name, work_directory, + process_name=process_name, stage="build", + ) + + +def world_scratch_dir(work_directory: Path, world_index: int, process_name: str) -> Path: + return _ensure_within( + work_directory / "worlds" / f"w{world_index}" / process_name, work_directory, + process_name=process_name, stage="spawn", + ) + + +def managed_engine_data_dir( + work_directory: Path, process_name: str, *, world_index: int | None +) -> Path: + """§0 fixes `/work/build//` and `/work/worlds/w//` but names no path for a + managed engine's own data directory. Per-world engines reuse the per-world scratch shape + (`world_index` given) so `reset`'s `datadir_copy` case (§4.2: "restore its data directory") + has an unambiguous per-world location. Job-shared engines (`world_index=None`) get a + job-level directory outside any single world's tree, since nothing about them is per-world — + `/work/managed//`, which §0 v1.8 added to the layout block (JC4, p5-round1-review). + """ + if world_index is None: + path = work_directory / "managed" / process_name + else: + return world_scratch_dir(work_directory, world_index, process_name) + return _ensure_within(path, work_directory, process_name=process_name, stage="spawn") + + +# --- §2b depends_on wait ------------------------------------------------------------------------- + + +def _readiness_probes_for_process( + manifest: EnvironmentBundleV2, process_name: str +) -> list[ReadinessProbeV2]: + """F8, p5-round1-review: returns EVERY declared probe backing this process, not just the + first-listed one — matching `healthy()`'s own definition of ready (§4 point 3: "declared + `readiness` probes," plural, unqualified). A process backing two capabilities used to be + treated as ready by `depends_on` the moment the first-listed probe passed, then immediately + reported unhealthy the instant `healthy()` ran, because the two functions disagreed about + what "ready" meant.""" + capability_slugs = { + slug + for slug, capability in manifest.capabilities.items() + if capability.service == process_name + } + return [probe for probe in manifest.readiness if probe.capability in capability_slugs] + + +def _tcp_probe(host: str, port: int, *, timeout: float = 0.75) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _probe_http(host: str, port: int, path: str | None, *, timeout: float = 1.0) -> bool: + url = f"http://{host}:{port}/{(path or '').lstrip('/')}" + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(url, timeout=timeout) as response: + response.read(256) + return 200 <= response.status < 400 + except (OSError, urllib.error.URLError, urllib.error.HTTPError): + return False + + +def _probe_postgres( + host: str, port: int, *, user: str | None = None, password: str | None = None, + dbname: str | None = None, timeout: float = 1.0, +) -> bool: + """F9, p5-round1-review: previously connected as `user="postgres", dbname="postgres"` — + neither exists, since the catalog's `initdb -U harness` creates only the `harness` role — and + treated almost any resulting `OperationalError` as "answered, therefore ready," including + `FATAL: the database system is starting up`, postgres's OWN response while still in recovery + (it binds its listen socket early). With real generated credentials threaded through, this + runs an actual `SELECT 1`, which a starting-up server cannot pass. Without credentials (no + generated role for this call site, or `psycopg` absent) it falls back to a bare TCP probe — + strictly weaker, but no longer claims false readiness from a substring match either. + """ + try: + import psycopg # type: ignore[import-not-found] + except ImportError: + return _tcp_probe(host, port, timeout=timeout) + if user is None or dbname is None: + return _tcp_probe(host, port, timeout=timeout) + try: + connection = psycopg.connect( + host=host, port=port, user=user, password=password, dbname=dbname, + connect_timeout=timeout, + ) + try: + connection.execute("SELECT 1") + finally: + connection.close() + return True + except psycopg.OperationalError: + return False + except Exception: + return False + + +def default_capability_prober( + *, protocol: CapabilityProtocol, host: str, port: int, path: str | None, + user: str | None = None, password: str | None = None, dbname: str | None = None, +) -> bool: + if protocol is CapabilityProtocol.POSTGRES: + return _probe_postgres(host, port, user=user, password=password, dbname=dbname) + if protocol is CapabilityProtocol.HTTP: + return _probe_http(host, port, path) + return _tcp_probe(host, port) + + +def _poll_until( + condition: Callable[[], bool], + *, + timeout: float, + interval: float, + clock: Callable[[], float], + sleep: Callable[[float], None], + timeout_error: Callable[[], Exception], +) -> None: + deadline = clock() + timeout + while True: + if condition(): + return + if clock() >= deadline: + raise timeout_error() + sleep(interval) + + +def _postgres_probe_credentials( + capability: CapabilityV2, + *, + world_index: int, + credentials: dict[str, EngineCredentials] | None, +) -> tuple[str | None, str | None, str | None]: + if capability.protocol is not CapabilityProtocol.POSTGRES: + return None, None, None + creds = (credentials or {}).get(capability.service) + if creds is None: + return None, None, None + return creds.username, creds.password, f"w{world_index}" + + +def wait_for_dependency( + manifest: EnvironmentBundleV2, + dependency_name: str, + *, + world_index: int, + port_plan: PortPlan, + spawned: SpawnedWorldProcess, + credentials: dict[str, EngineCredentials] | None = None, + prober: CapabilityProber = default_capability_prober, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """§2b: "the dependent starts only after the dependency's capability `readiness` probe passes + (or its `started_check`, or immediately after spawn if it has neither)." Priority order per + the contract's own parenthetical: a declared capability readiness probe first; `started_check` + only when the dependency backs no such capability (and only `SourceProcess` ever carries one — + `ManagedProcess` has no `started_check` field); otherwise return immediately, since spawn + already happened before this is ever called. + + A dependency backing more than one capability must pass EVERY declared probe (F8), matching + `healthy()`'s own definition — the combined timeout is the longest of them (no probe is cut + off early), the combined poll interval is the tightest of them (no probe is polled less often + than it declared). `credentials` (F9) lets a postgres probe run a real `SELECT 1` instead of + a bare TCP connect; omitted, it degrades to the same TCP-only check as before. + """ + probes = _readiness_probes_for_process(manifest, dependency_name) + if probes: + port = port_plan.port_for(dependency_name, world_index) + + def probe_ready() -> bool: + for probe in probes: + capability = manifest.capabilities[probe.capability] + user, password, dbname = _postgres_probe_credentials( + capability, world_index=world_index, credentials=credentials + ) + if not prober( + protocol=capability.protocol, host="localhost", port=port, path=probe.path, + user=user, password=password, dbname=dbname, + ): + return False + return True + + combined_timeout = max(probe.timeout_seconds for probe in probes) + _poll_until( + probe_ready, + timeout=combined_timeout, + interval=min(probe.interval_seconds for probe in probes), + clock=clock, + sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "depends_on", + "depends_on_timeout", + f"{dependency_name}: readiness probe did not pass within {combined_timeout}s", + process=dependency_name, + ), + ) + return + + processes_by_name = {process.name: process for process in manifest.processes} + dependency = processes_by_name[dependency_name] + started_check = dependency.started_check if isinstance(dependency, SourceProcess) else None + if started_check is None: + return # neither a readiness probe nor a started_check — ready immediately after spawn. + + if started_check.port: + # §2b (v1.8): the field only SELECTS the port-probe variant — the dialed port is always + # the dependency's own allocated one (`port_plan.port_for`, honoring `fixed_port`), never + # a literal read from the manifest (F4, p5-round1-review). A literal cannot be correct for + # more than one world, since the formula port differs by `world_index`. + port = port_plan.port_for(dependency_name, world_index) + + def condition() -> bool: + return _tcp_probe("localhost", port) + else: + marker = started_check.log_marker + + def condition() -> bool: + return marker in spawned.handle.captured_output() + + _poll_until( + condition, + timeout=started_check.timeout_seconds, + interval=0.25, + clock=clock, + sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "depends_on", + "depends_on_timeout", + f"{dependency_name}: started_check did not pass within " + f"{started_check.timeout_seconds}s", + process=dependency_name, + ), + ) + + +def _topological_order(manifest: EnvironmentBundleV2) -> list[str]: + """Dependencies before dependents. `preflight_bundle` (`_verify_depends_on`) already rejects + cycles and unknown names before this ever runs, so a DAG is assumed here, not re-verified.""" + graph = {process.name: list(process.depends_on) for process in manifest.processes} + order: list[str] = [] + visited: set[str] = set() + + def visit(name: str) -> None: + if name in visited: + return + visited.add(name) + for dependency_name in graph[name]: + visit(dependency_name) + order.append(name) + + for name in graph: + visit(name) + return order + + +# --- per-world spawn orchestration -------------------------------------------------------------- + + +@dataclass(frozen=True) +class SpawnContext: + """The caller-supplied dependencies `spawn_world` needs, grouped so the per-call signature + stays to (manifest, world_index, shared_handles) — Phase 6's `provision()` builds one of + these per job and calls `spawn_world` once per world index inside its own reconciliation + loop; that loop, idempotency across retries, and `EnvironmentRuntime` state assembly across + all `instances` worlds are its job, not this function's. + """ + + work_directory: Path + port_plan: PortPlan + credentials: dict[str, EngineCredentials] + secret_values: dict[str, str] + secret_purposes: dict[str, str] + runner: ProcessRunner = default_process_runner + # Shared by `build_process_trees`' `build_commands` steps and `spawn_managed_process`'s + # postgres `initdb` bootstrap — both are "run one synchronous step, check its exit code." + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run + prober: CapabilityProber = default_capability_prober + copy: Callable[[Path, Path], None] | None = None + # F1, p5-round1-review: threaded to every build/spawn call this context drives. + # `require_declared_user=False` is the local-lane default (no `svc-*` accounts on a dev box); + # a hosted caller sets it `True` so a snapshot that somehow lacks a declared user fails typed + # instead of silently running unprivileged. + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver + require_declared_user: bool = False + chown: Callable[[Path, int, int], None] = _default_chown + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS + + +@dataclass +class WorldSpawnResult: + handles: dict[str, SpawnedWorldProcess] + endpoints: dict[str, RuntimeEndpoint] + + +def spawn_world( + manifest: EnvironmentBundleV2, + *, + world_index: int, + context: SpawnContext, + shared_handles: dict[str, SpawnedWorldProcess] | None = None, +) -> WorldSpawnResult: + """Spawns every process for ONE world, in `depends_on` dependency order, waiting on each + dependency's readiness before starting its dependent (§2b). `shared_handles` carries + already-running job-shared managed engines (from an earlier world's call) so they are reused, + never respawned, for `world_index > 0` — the caller is expected to thread the same dict + through every `spawn_world` call for one job. Source process build trees must already exist + (via `build_process_tree`, called once per process before any world is spawned) — this + function only creates per-world scratch directories, never a build tree. + """ + endpoints = build_endpoints( + manifest, + world_index=world_index, + port_plan=context.port_plan, + credentials=context.credentials, + ) + configuration_addresses = configuration_addresses_from_endpoints(endpoints) + processes_by_name = {process.name: process for process in manifest.processes} + handles: dict[str, SpawnedWorldProcess] = dict(shared_handles or {}) + + for name in _topological_order(manifest): + if name in handles: + continue # a job-shared managed engine already running from an earlier world. + process = processes_by_name[name] + for dependency_name in process.depends_on: + wait_for_dependency( + manifest, + dependency_name, + world_index=world_index, + port_plan=context.port_plan, + spawned=handles[dependency_name], + credentials=context.credentials, + prober=context.prober, + ) + if isinstance(process, ManagedProcess): + data_dir = managed_engine_data_dir( + context.work_directory, + name, + world_index=None if context.port_plan.is_job_shared(name) else world_index, + ) + handle = spawn_managed_process( + process, + port=context.port_plan.port_for(name, world_index), + data_dir=data_dir, + credentials=context.credentials.get(name), + runner=context.runner, + sync_run=context.sync_run, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + else: + handle = spawn_source_process( + process, + build_dir=build_tree_dir(context.work_directory, name), + world_dir=world_scratch_dir(context.work_directory, world_index, name), + world_index=world_index, + port_plan=context.port_plan, + configuration_addresses=configuration_addresses, + secret_values=context.secret_values, + secret_purposes=context.secret_purposes, + runner=context.runner, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + handles[name] = handle + return WorldSpawnResult(handles=handles, endpoints=endpoints) + + +def build_process_trees( + manifest: EnvironmentBundleV2, *, source_root: Path, context: SpawnContext +) -> dict[str, Path]: + """Builds every `source` process's tree once — the caller invokes this exactly once per job, + before the first `spawn_world` call for any world (§2b: `build_commands` run "once per + job").""" + return { + process.name: build_process_tree( + process, + source_root=source_root, + build_root=context.work_directory / "build", + run=context.sync_run, + copy=context.copy, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + build_step_timeout_seconds=context.build_step_timeout_seconds, + ) + for process in manifest.processes + if isinstance(process, SourceProcess) + } + + +# --- §4.3 healthy() -------------------------------------------------------------------------- + + +def _split_host_port(address: str) -> tuple[str, int]: + parsed = urlsplit(address) + return parsed.hostname or "localhost", parsed.port or 0 + + +def _postgres_credentials_from_address(address: str) -> tuple[str | None, str | None, str | None]: + """F9: the rendered `endpoint.address` for a postgres capability already carries the + generated role/password/database (`postgresql://harness:@localhost:/w`) — parsed + back out rather than threading `EngineCredentials` separately into `probe_runtime_health`, + which only ever sees `EnvironmentRuntime`, not the job's credential map.""" + parsed = urlsplit(address) + return parsed.username, parsed.password, (parsed.path.lstrip("/") or None) + + +def probe_runtime_health( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber = default_capability_prober, +) -> bool: + """§4 point 3: "`healthy` = declared `readiness` probes, not 'process is running.'" Every + declared `readiness` entry for a capability this runtime actually exposes must pass; a + capability with no readiness entry declared carries no health obligation (§2b: "Health/ + readiness is otherwise declared ONLY in the capability-level `readiness` section").""" + for probe in manifest.readiness: + endpoint = runtime.endpoints.get(probe.capability) + capability = manifest.capabilities.get(probe.capability) + if endpoint is None or capability is None: + return False + host, port = _split_host_port(endpoint.address) + user = password = dbname = None + if capability.protocol is CapabilityProtocol.POSTGRES: + user, password, dbname = _postgres_credentials_from_address(endpoint.address) + if not prober( + protocol=capability.protocol, host=host, port=port, path=probe.path, + user=user, password=password, dbname=dbname, + ): + return False + return True + + +async def healthy( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber = default_capability_prober, +) -> bool: + """Async wrapper matching §4's `RuntimeProvider.healthy` shape (`runtime.py`'s own + `LocalComposeRuntimeProvider.healthy` uses the same `asyncio.to_thread` idiom). + + §3's `state` transitions are fixed: `preparing->ready`, `ready->unhealthy`, and + `unhealthy->ready` ONLY via re-provision reconcile (§4 point 1: "a sick world mid-job is + recovered by calling `provision` again"). `healthy()` is not a reconcile (F6, p5-round1- + review) — it may only ever DEMOTE `runtime.state`, never promote it. `ready` stays `ready` + while still healthy; `unhealthy`/`stopped` are cleared only by `provision()`/`close()`, never + by a passing probe alone (a world whose sentinel was never re-proved after coming back up must + not silently re-enter the pool). + """ + import asyncio + + is_healthy = await asyncio.to_thread(probe_runtime_health, manifest, runtime, prober=prober) + if not is_healthy: + runtime.state = RuntimeState.UNHEALTHY + elif runtime.state is RuntimeState.PREPARING: + runtime.state = RuntimeState.READY + return is_healthy + + +__all__ = [ + "CapabilityProber", + "EngineCredentials", + "EnvironmentRuntime", + "PopenProcess", + "PortPlan", + "ProcessRunner", + "ProcessRuntimeError", + "RuntimeEndpoint", + "RuntimeState", + "SpawnContext", + "SpawnedProcess", + "SpawnedWorldProcess", + "WorldSpawnResult", + "build_endpoints", + "build_process_tree", + "build_process_trees", + "build_tree_dir", + "configuration_addresses_from_endpoints", + "default_capability_prober", + "default_process_runner", + "default_user_resolver", + "generate_engine_credentials", + "healthy", + "managed_engine_data_dir", + "new_runtime_id", + "plan_ports", + "probe_runtime_health", + "rabbitmq_daemon_argv", + "rabbitmq_daemon_env", + "redis_daemon_argv", + "render_capability_address", + "render_environment", + "render_template", + "postgres_bootstrap_argv", + "postgres_daemon_argv", + "select_process_secrets", + "spawn_managed_process", + "spawn_source_process", + "spawn_world", + "wait_for_dependency", + "world_scratch_dir", +] diff --git a/tests/harness/test_bundle_v2.py b/tests/harness/test_bundle_v2.py index 7494e430..c581dbf9 100644 --- a/tests/harness/test_bundle_v2.py +++ b/tests/harness/test_bundle_v2.py @@ -402,6 +402,32 @@ def test_unknown_field_on_a_process_entry_is_rejected() -> None: ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "mounts": ["/data"]}) +@pytest.mark.parametrize( + "bad_name", ["/etc", "../../etc", "Tools-Api", "tools_api!", "-leading-dash"], + ids=["absolute", "traversal", "uppercase", "punctuation", "leading-dash"], +) +def test_a_process_name_outside_the_closed_pattern_is_rejected(bad_name: str) -> None: + """F3, p5-round1-review — BLOCKER, model layer. `name` is path-joined into + `/work/build//` and `/work/worlds/w//` verbatim (§2b) — an unvalidated name + used to make `Path("/work/build") / "/etc"` collapse to `Path("/etc")`, which the provisioner + then `rmtree`'d as svc-control. Both process classes carry the same + `^[a-z0-9][a-z0-9_-]*$` pattern (§0 v1.8); `process_runtime.py`'s own `_ensure_within` is the + defense-in-depth backstop for a caller that bypasses this model layer entirely (see + `test_process_runtime.py`'s `test_build_tree_dir_rejects_a_name_that_escapes_the_work_directory` + and siblings — "tests both layers," per the worklist).""" + with pytest.raises(ValidationError, match="process_name_invalid"): + ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "name": bad_name}) + with pytest.raises(ValidationError, match="process_name_invalid"): + SourceProcess.model_validate({**AGENT_PROCESS_EXAMPLE, "name": bad_name}) + + +def test_process_names_matching_every_2b_example_are_accepted() -> None: + """Every example name in §2b's own JSON (`postgres`, `tools-api`, `agent`) must keep working — + the pattern is closed, not merely restrictive.""" + for name in ("postgres", "tools-api", "agent"): + assert ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "name": name}).name == name + + # --- §2a/§2b/§2d rules the model owns on its own, exercised at manifest scope ---------------- @@ -464,9 +490,16 @@ def test_an_unresolved_control_service_is_rejected() -> None: def test_a_control_service_resolving_to_a_managed_engine_is_rejected() -> None: """N9 (p4-round2-review): `control_service` names the agent-side service the world handle and - evidence seam attach to (§2a) — a datastore in that role is incoherent. Before this check, the - `ManagedProcess` branch of the user-assignment loop below ran first and expected `svc-data` - for it, which `postgres` already has, so the bundle silently loaded.""" + evidence seam attach to (§2a) — a datastore in that role is incoherent. For THIS fixture, + pre-fix, the user-assignment loop below still caught it, just under the wrong code: `postgres` + got its expected `svc-data` (fine), but `agent` — carrying `user: "svc-agent"` from + `AGENT_PROCESS_EXAMPLE` — is no longer `control_service`, so the loop demands `svc-tools` for + it instead and raises `user_assignment_invalid: agent must be svc-tools, got svc-agent` (B1, + p4-round3-review). The genuine silent-load case needs a bundle where no source process claims + `svc-agent` at all (e.g. only `postgres` + `tools-api`) — every expectation is then satisfied + and the bundle loads with no agent-side process in it. This test still discriminates the fix + either way: the pre-fix code (`user_assignment_invalid`) does not match the post-fix code + (`control_service_unresolved`), so it goes red on a revert.""" manifest = { **FULL_MANIFEST_EXAMPLE, "runtime": {**RUNTIME_EXAMPLE, "control_service": "postgres"}, diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py index 7a7046bc..2fa3b13b 100644 --- a/tests/harness/test_process_preflight.py +++ b/tests/harness/test_process_preflight.py @@ -1,4 +1,4 @@ -"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.7. +"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.8. Every checklist item gets at least one rejection test carrying its named code, plus one clean accept-lane run of the full checklist. Bundles are built as real directories under `tmp_path` with @@ -9,13 +9,20 @@ from __future__ import annotations +import ast import hashlib +import inspect import json +import re from pathlib import Path from typing import Any, Callable import pytest +from fi.alk.harness import bundle as bundle_module +from fi.alk.harness import bundle_v2 as bundle_v2_module +from fi.alk.harness import process_preflight as process_preflight_module +from fi.alk.harness import process_runtime as process_runtime_module from fi.alk.harness.bundle_v2 import ( BUNDLE_V2_SCHEMA_VERSION, EnvironmentBundleV2, @@ -481,6 +488,37 @@ def mutate(body: dict[str, Any]) -> dict[str, Any]: preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) +@pytest.mark.parametrize("colliding_port", [14000, 14099, 15000, 15799]) +def test_a_fixed_port_colliding_with_a_port_formula_band_is_rejected( + tmp_path: Path, colliding_port: int +) -> None: + """F11, p5-round1-review: `fixed_port` forces effective parallelism to 1, but the literal + value was never checked against the provisioner's own port-formula bands — a bundle declaring + `fixed_port: 14000` collides with a job-shared engine at ordinal 0, and the failure mode is an + opaque bind error inside a customer process, not a bundle rejection. `fixed_port_reserved` has + no §2e table entry yet (flagged for the owner in `PreflightError`'s own docstring) — the + S3 containment test below carries the same flag.""" + + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = colliding_port + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="fixed_port_reserved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_fixed_port_outside_both_bands_is_accepted(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = 9000 + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + assert preflight_bundle( + tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS + ) is None + + def test_a_postgres_capability_with_no_store_entry_is_rejected(tmp_path: Path) -> None: def mutate(body: dict[str, Any]) -> dict[str, Any]: body["capabilities"]["other_db"] = { @@ -705,6 +743,30 @@ def test_a_compose_bundle_with_an_unlisted_file_reports_compose_not_hosted( preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) +def test_a_compose_bundle_with_a_changed_listed_file_reports_compose_not_hosted( + tmp_path: Path, +) -> None: + """B2 (p4-round3-review): the fixture above only pins the gate above item 2 — its sole + competing violation (`compose.yaml` unlisted) never reaches item 1, since the bundle is + otherwise digest-consistent. This mutates a LISTED file's bytes after sealing, without + resealing, so item 1 would otherwise raise `bundle_file_changed` — proving the gate wins + above item 1 too, not just item 2.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + (tmp_path / "compose.yaml").write_bytes(b"services: {}\n") + (tmp_path / "db" / "schema.sql").write_bytes(b"MUTATED") # still listed in files[]; unsealed. + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + # --- kind: external ------------------------------------------------------------------------------ @@ -738,3 +800,232 @@ def test_kind_external_skips_the_process_block_but_still_enforces_resource_sanit # Item 7 still runs regardless of runtime kind. with pytest.raises(PreflightError, match="parallelism_out_of_range"): preflight_bundle(tmp_path, manifest, parallelism=99, secret_refs={}) + + +# --- §2e-table containment (B3/S3, p4-round3-review) --------------------------------------------- +# +# `PreflightError`'s own docstring claims "every code this module raises is in that [§2e] table" — +# true today, per p4-round3-review's part-B finding, but mechanically unenforced: a future +# `raise ValueError("some_new_code: ...")` anywhere in the model layer would silently leak an +# unlisted code across a seam the contract calls closed, and nothing short of re-deriving the claim +# by hand (as that review did) would catch it. This makes the claim a running test instead. +# +# Approach (grep-free, as the worklist asked): walk each module's SOURCE TEXT with `ast`, not +# `grep` — a plain substring/regex scan over raw text can't distinguish an actual `raise +# PreflightError(...)`/`raise ValueError(...)` call from a code fragment inside a docstring or a +# comment quoting one. Two extraction shapes: +# - `PreflightError(...)`: the whole first positional argument, when it is a literal string, IS +# the code (every call in this codebase spells it that way) — `_translate_validation_error`'s +# `extra_forbidden` branch RETURNS its `PreflightError(...)` rather than raising it directly +# (the caller does `raise _translate_validation_error(exc) from exc`), so the walk matches +# every `Call` node by callee name, not only ones sitting directly under a `Raise`. +# - `ValueError(...)` (the model layer): the code is only the LEADING token of the message, up +# to `:` or end-of-string — recovered by taking the static leading text of the first argument +# (a plain string constant, an f-string's first literal segment, or the left operand of a `+` +# concatenation — every shape actually used in `bundle_v2.py`/`bundle.py`) and applying the +# same `[a-z][a-z0-9_]*` leading-token regex `_translate_validation_error`'s own runtime +# fallback uses. A `ValueError` whose message has no static leading text at all (fully +# dynamic) would be skipped rather than mis-coded — none exist in these three modules today. +# +# Scope judgement call: `bundle.py` also carries `EnvironmentBundle`/`BundleRuntime` (v1's OWN, +# separate model classes) with their own, unrelated `ValueError` raises (e.g. plain prose with no +# code prefix at all, `bundle_digest_invalid`/`bundle_schema_unsupported` reused as v1's own +# names) — none of that is reachable from `EnvironmentBundleV2.model_validate(...)`'s validation +# tree, since v1 and v2 are disjoint model hierarchies (`bundle_v2.py`'s own docstring: "v1 stays +# untouched"). The only `bundle.py` surface v2 actually calls into is the two helper functions +# `bundle_v2.py` imports and its own validators invoke — `_safe_relative`, `_reject_secret_values` +# — so this test walks exactly those two functions' source (`inspect.getsource`), not the whole +# file; walking the whole file would both over-include unreachable v1 codes and risk masking a +# real gap behind noise from a hierarchy this claim was never about. + + +_CODE_LEADING_TOKEN = re.compile(r"^([a-z][a-z0-9_]*)(?::|$)") + + +def _static_leading_text(node: ast.expr) -> str | None: + """The static leading text of a raise-argument expression, or `None` if it has none at all + (a fully dynamic value, e.g. a bare name or an f-string starting with a `{...}`).""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr) and node.values and isinstance(node.values[0], ast.Constant): + return str(node.values[0].value) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return _static_leading_text(node.left) + return None + + +def _raised_codes(source_text: str, callee_name: str, *, whole_first_argument: bool) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(source_text)): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Name) and func.id == callee_name): + continue + if not node.args: + continue + if whole_first_argument: + first = node.args[0] + if isinstance(first, ast.Constant) and isinstance(first.value, str): + codes.add(first.value) + continue + leading = _static_leading_text(node.args[0]) + if leading is None: + continue + match = _CODE_LEADING_TOKEN.match(leading) + if match: + codes.add(match.group(1)) + return codes + + +# §2e's closed failure-code table (v1.8), transcribed verbatim — the single source of truth every +# raised code is checked against. Split exactly as the contract text splits it, purely for +# reviewability against the spec; the test below treats it as one flat set. +# +# `fixed_port_reserved` (F11, p5-round1-review) is the one entry NOT actually in the frozen v1.8 +# table — flagged identically in `PreflightError`'s own docstring. The rule it guards (a +# `fixed_port` aliasing the provisioner's own port-formula bands) is real; the table predates it. +# Recorded here, transcribed alongside the real entries rather than hidden in a second set, so +# this test still does its job for every OTHER code — the one exception is a known, owner-facing +# gap, not silent drift. +_SECTION_2E_CONTRACT_RULE_CODES = frozenset({ + "compose_not_hosted", "engine_unsupported", "no_sql_store", "seed_missing", + "seed_strategy_unsupported", "sentinel_shape_mismatch", "store_protocol_unsupported", + "capability_engine_mismatch", "store_service_not_managed", "reserved_name", + "unknown_placeholder", "unknown_field", "secret_in_bundle", "secret_unclaimed", + "secret_missing", "build_requires_root", "user_assignment_invalid", + "configuration_name_duplicate", "configuration_name_required", + "configuration_name_reserved", "sentinel_shape_invalid", "capability_unresolved", + "service_unresolved", "control_service_unresolved", "process_name_duplicate", + "inputs_digest_mismatch", + "fixed_port_reserved", # NOT in the frozen v1.8 table yet — see the note above. +}) +_SECTION_2E_MECHANICAL_CODES = frozenset({ + "bundle_schema_unsupported", "bundle_manifest_invalid", "bundle_manifest_drifted", + "bundle_digest_mismatch", "bundle_digest_invalid", "inputs_digest_invalid", + "file_sha256_invalid", "source_digest_invalid", "bundle_file_missing", + "bundle_file_changed", "bundle_file_unlisted", "bundle_symlink_forbidden", + "bundle_path_unsafe", "depends_on_unresolved", "depends_on_cycle", "seed_file_missing", + "seed_file_unlisted", "process_count_exceeded", "parallelism_out_of_range", + "evidence_seam_required", "processes_required", "processes_and_seed_forbidden", + "document_only_for_compose", "compose_runtime_requires_document", + "build_command_step_empty", "started_check_requires_exactly_one_of_port_or_log_marker", + "resolved_secret_forbidden", "capability_slug_invalid", + "process_name_invalid", # §0/§2b v1.8: the process-`name` pattern rule. +}) +_SECTION_2E_CODES = _SECTION_2E_CONTRACT_RULE_CODES | _SECTION_2E_MECHANICAL_CODES + + +def test_every_code_these_modules_can_raise_is_in_the_closed_section_2e_table() -> None: + raised: set[str] = set() + raised |= _raised_codes( + Path(inspect.getfile(process_preflight_module)).read_text(encoding="utf-8"), + "PreflightError", whole_first_argument=True, + ) + raised |= _raised_codes( + Path(inspect.getfile(bundle_v2_module)).read_text(encoding="utf-8"), + "ValueError", whole_first_argument=False, + ) + raised |= _raised_codes( + inspect.getsource(bundle_module._safe_relative), "ValueError", whole_first_argument=False + ) + raised |= _raised_codes( + inspect.getsource(bundle_module._reject_secret_values), + "ValueError", whole_first_argument=False, + ) + unlisted = raised - _SECTION_2E_CODES + assert not unlisted, f"raised but not in §2e's table: {sorted(unlisted)}" + + +def test_the_extraction_itself_finds_a_nonempty_set_in_every_source() -> None: + """Guards the test above against a false pass from a broken extractor (e.g. an import that + silently resolves to the wrong file, or a callee-name typo) — each source individually must + contribute at least one code, not just the union as a whole.""" + preflight_codes = _raised_codes( + Path(inspect.getfile(process_preflight_module)).read_text(encoding="utf-8"), + "PreflightError", whole_first_argument=True, + ) + bundle_v2_codes = _raised_codes( + Path(inspect.getfile(bundle_v2_module)).read_text(encoding="utf-8"), + "ValueError", whole_first_argument=False, + ) + assert "unknown_field" in preflight_codes # the `return`-not-`raise` case (see module note). + assert "compose_not_hosted" in preflight_codes + assert len(bundle_v2_codes) > 10 + assert "bundle_path_unsafe" in _raised_codes( + inspect.getsource(bundle_module._safe_relative), "ValueError", whole_first_argument=False + ) + assert "resolved_secret_forbidden" in _raised_codes( + inspect.getsource(bundle_module._reject_secret_values), + "ValueError", whole_first_argument=False, + ) + + +# --- §2f-table containment (v1.8 addition) -------------------------------------------------------- +# +# v1.8 gave `process_runtime.py` its own closed table (§2f) for the subset of `ProcessRuntimeError` +# codes that cross the outbound seam. Same containment argument as §2e above, extended to the +# module the new table covers — worklist item: `unsupported_capability_protocol` and +# `source_tree_unavailable` are new in this fix pass and must show up here or the test (correctly) +# fails. `ProcessRuntimeError(stage, code, message, ...)` carries its code at positional index 1, +# not index 0 (`PreflightError`'s shape), so this is a distinct extraction, not a reuse of +# `_raised_codes`. + + +def _raised_codes_at_index(source_text: str, callee_name: str, *, index: int) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(source_text)): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Name) and func.id == callee_name): + continue + if len(node.args) <= index: + continue + argument = node.args[index] + if isinstance(argument, ast.Constant) and isinstance(argument.value, str): + codes.add(argument.value) + return codes + + +# §2f's closed table (v1.8), transcribed verbatim. +_SECTION_2F_CODES = frozenset({ + "source_tree_unavailable", "build_failed", "runtime_unsupported", "spawn_failed", + "depends_on_timeout", "unsupported_capability_protocol", +}) +# `ProcessRuntimeError` also raises codes that are deliberately INTERNAL-only — each marks a +# precondition `preflight_bundle` should already have made impossible (a placeholder token or a +# missing credential preflight itself should have caught), so by the module's own docstring these +# "never cross the outbound seam directly" and have no §2f entry to begin with. Excluded from +# CONTAINMENT, not from extraction — a genuinely new internal code still surfaces in the raised +# set for a human to classify, since only these two documented names are exempted. +_INTERNAL_ONLY_RUNTIME_CODES = frozenset({ + "internal_unknown_placeholder", "internal_missing_credentials", +}) + + +def test_every_section_2f_code_process_runtime_raises_is_in_the_closed_table() -> None: + raised = _raised_codes_at_index( + Path(inspect.getfile(process_runtime_module)).read_text(encoding="utf-8"), + "ProcessRuntimeError", index=1, + ) + # `process_name_invalid` (F3's defense-in-depth path-containment check, `_ensure_within`) is + # not a NEW §2f code — it deliberately reuses the EXISTING §2e model-layer code as a backstop + # for when the model layer is bypassed, so `_SECTION_2E_CODES` is a legitimate source too, not + # just §2f's own six entries. + unlisted = raised - _SECTION_2F_CODES - _SECTION_2E_CODES - _INTERNAL_ONLY_RUNTIME_CODES + assert not unlisted, ( + f"raised but not in §2f's table (nor §2e's, nor a documented internal-only code): " + f"{sorted(unlisted)}" + ) + + +def test_the_section_2f_extraction_itself_finds_a_nonempty_set() -> None: + raised = _raised_codes_at_index( + Path(inspect.getfile(process_runtime_module)).read_text(encoding="utf-8"), + "ProcessRuntimeError", index=1, + ) + assert "build_failed" in raised + assert "spawn_failed" in raised + assert "source_tree_unavailable" in raised + assert "unsupported_capability_protocol" in raised diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py new file mode 100644 index 00000000..203b73ed --- /dev/null +++ b/tests/harness/test_process_runtime.py @@ -0,0 +1,1821 @@ +"""The execution half of the provisioner (`process_runtime.py`), per `hosted-execution-seams.md` +v1.8 §2b/§3/§4. Manifests here are built directly through `EnvironmentBundleV2.model_validate` — +no digest sealing, no on-disk bundle — since every rule under test needs nothing but the parsed +model's own field values plus the job-supplied inputs (instances, secrets, credentials). Digest +and file-content verification are `test_process_preflight.py`'s job, not this module's. + +No Docker, no real postgres/redis/rabbitmq anywhere: managed-engine spawn is exercised only +through fakes (`ProcessRunner`, `sync_run`), since §0 assumes those binaries are on the snapshot's +PATH and this test lane must not require that. Real subprocess spawn IS exercised, through tiny +`python3 -c ...` stand-in scripts — proving `default_process_runner` genuinely spawns, captures +output, and reports liveness, without any engine-specific plumbing. + +P5's fix pass (F1/F2/F3/F7) made real filesystem operations security-relevant, so those are +exercised directly against real symlinks/tmp dirs where a fake would hide the exact class of bug +being fixed. `os.chown`/`Popen(user=...)` to a FOREIGN uid needs root, which this lane does not +have and must not require — those paths are verified STRUCTURALLY: a fake `chown`/`user_resolver` +is injected and the call it WOULD make is asserted, rather than performing the real privileged +syscall. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +import time +import types +from pathlib import Path +from typing import Any, Callable + +import pytest + +from fi.alk.harness.bundle import CapabilityProtocol +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + CapabilityV2, + EnvironmentBundleV2, + ProcessUser, + SourceProcess, +) +from fi.alk.harness import process_runtime as pr + +# --- shared manifest builder ----------------------------------------------------------------- +# +# One postgres store (job-shared, `template_database`), one `tools-api` http-capability source +# process, one `agent` control-service source process claiming the job's only `target_provider` +# secret — the same shape `test_process_preflight.py` uses, so port/ordinal numbers a reader +# already knows from that file carry over here. + + +def _body(mutate: Callable[[dict[str, Any]], dict[str, Any]] | None = None) -> dict[str, Any]: + body: dict[str, Any] = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "digest": "sha256:" + "0" * 64, + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": [], + }, + { + "name": "tools-api", "kind": "source", "working_directory": "services/tools-api", + "build_commands": [["npm", "ci"]], "run_command": ["node", "server.js"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "PORT": "{{PORT_tools-api}}", + "TMPDIR": "{{WORLD_DIR}}", + }, + "secret_purposes": [], "user": "svc-tools", "depends_on": ["postgres"], + }, + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python3", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "TOOLS_API_URL": "{{TOOLS_API_URL}}", + "NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], "user": "svc-agent", + "depends_on": ["postgres", "tools-api"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", "service": "postgres", "configuration_name": "DATABASE_URL", + }, + "tools": { + "protocol": "http", "service": "tools-api", "configuration_name": "TOOLS_API_URL", + }, + }, + "readiness": [ + { + "capability": "tools", "path": "/health", "timeout_seconds": 5, + "interval_seconds": 0.1, + }, + ], + "seed": { + "stores": [ + { + "capability": "database", "migrations": [], "seed_files": [], + "baseline": { + "strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64 + }, + "sentinel": {"query": "SELECT 1", "expected": "1"}, + } + ] + }, + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64 + }, + "metadata": {}, + } + return mutate(body) if mutate is not None else body + + +def _manifest( + mutate: Callable[[dict[str, Any]], dict[str, Any]] | None = None, +) -> EnvironmentBundleV2: + return EnvironmentBundleV2.model_validate(_body(mutate)) + + +def _source_process(**overrides: Any) -> SourceProcess: + fields: dict[str, Any] = { + "name": "svc", "kind": "source", "working_directory": ".", "build_commands": [], + "run_command": ["python3", "-c", "pass"], "environment": {}, "secret_purposes": [], + "user": ProcessUser.SVC_TOOLS, "depends_on": [], + } + fields.update(overrides) + return SourceProcess(**fields) + + +def _solo_port_plan(process_name: str, *, ordinal: int = 0) -> pr.PortPlan: + """A single-process `PortPlan` for tests that spawn a `_source_process()` standalone, off the + shared `_manifest()` topology — `spawn_source_process` always looks its own process up in the + plan it is given, so the plan must actually know that process's name.""" + return pr.PortPlan( + ordinals={process_name: ordinal}, job_shared=frozenset(), fixed_ports={}, + effective_instances=1, degraded_reason=None, + ) + + +class FakeHandle: + """A `SpawnedProcess` fake: no real subprocess, just a captured-output buffer.""" + + def __init__(self, output: str = "") -> None: + self._output = output + self.terminated = False + + def is_running(self) -> bool: + return not self.terminated + + def captured_output(self) -> str: + return self._output + + def terminate(self) -> None: + self.terminated = True + + +class _FakePasswd: + """Stands in for `pwd.struct_passwd` — only `pw_uid`/`pw_gid` are ever read by this module.""" + + def __init__(self, uid: int, gid: int) -> None: + self.pw_uid = uid + self.pw_gid = gid + + +def _fake_user_resolver(known: dict[str, tuple[int, int]]) -> Callable[[str], _FakePasswd | None]: + def resolver(username: str) -> _FakePasswd | None: + if username in known: + uid, gid = known[username] + return _FakePasswd(uid, gid) + return None + + return resolver + + +def _reap(handle: Any, *, attempts: int = 50, interval: float = 0.05) -> None: + """Waits for a real spawned subprocess to exit. `pytest.fail`s loudly on exhaustion rather + than falling through to an assertion that would blame the wrong thing — a hung reap here means + the child never exited, not that its output was wrong.""" + for _ in range(attempts): + if not handle.is_running(): + return + time.sleep(interval) + else: + pytest.fail(f"subprocess did not exit within {attempts * interval}s") + + +# --- §2b port allocation ------------------------------------------------------------------------ + + +def test_ordinals_follow_the_authored_processes_array_order() -> None: + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.ordinals == {"postgres": 0, "tools-api": 1, "agent": 2} + + +def test_a_template_database_managed_engine_is_job_shared() -> None: + """§2b: `template_database` -> once per job; `14000 + ordinal`, the same in every world.""" + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.is_job_shared("postgres") + assert plan.port_for("postgres", 0) == 14000 + assert plan.port_for("postgres", 2) == 14000 + + +def test_source_processes_use_the_per_world_stride_formula() -> None: + """§2b: `15000 + 100*world_index + ordinal`.""" + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.port_for("tools-api", 0) == 15001 + assert plan.port_for("tools-api", 1) == 15101 + assert plan.port_for("agent", 2) == 15202 + + +def test_a_datadir_copy_managed_engine_is_per_world_not_job_shared() -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [ + {**body["seed"]["stores"][0], "baseline": { + "strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64 + }}, + ]}} + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("postgres") + assert plan.port_for("postgres", 0) == 15000 + assert plan.port_for("postgres", 1) == 15100 + + +def test_a_managed_engine_with_no_seed_entry_at_all_defaults_to_per_world() -> None: + """§2b: "per-world is the only safe default" when a managed engine has no `seed.stores` + entry — a shared engine one world reset could otherwise corrupt for the others. + + T4, p5-round1-review: a REDIS capability with no store entry (§2c: implicitly `empty`, safe + with no `seed` block at all) rather than the old postgres one — a postgres capability with no + store entry is preflight-*invalid* (`seed_missing`), so the old fixture proved this branch on + a manifest that could never reach the provisioner for real. `postgres`'s own `seed.stores` + entry (`template_database`) is left in place, unrelated to the branch under test. + """ + manifest = _manifest( + lambda body: { + **body, + "processes": [*body["processes"], { + "name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": [], + }], + "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": None}, + }, + } + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("cache") + assert plan.port_for("cache", 0) != plan.port_for("cache", 1) + + +def test_the_empty_baseline_strategy_managed_engine_is_per_world() -> None: + """T5, p5-round1-review: `empty` is one of §2b's four instancing branches + (`template_database` job-shared; `datadir_copy`, `empty`, and no-seed-entry-at-all all + per-world) and had no direct test — only `datadir_copy` and the no-entry case were covered. + Redis is the only catalog engine whose strategies include `empty` + (`bundle_v2._ENGINE_STRATEGIES`).""" + manifest = _manifest( + lambda body: { + **body, + "processes": [*body["processes"], { + "name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": [], + }], + "capabilities": { + **body["capabilities"], + "cache": { + "protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL", + }, + }, + "seed": {"stores": [ + body["seed"]["stores"][0], + { + "capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "_seeded", "expected": "1"}, + }, + ]}, + } + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("cache") + assert plan.port_for("cache", 0) == 15003 # ordinal 3 (postgres,tools-api,agent,cache), world 0 + assert plan.port_for("cache", 1) == 15103 # ordinal 3, world 1 + + +def test_fixed_port_is_honored_exactly_and_forces_effective_instances_to_one() -> None: + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], + {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + plan = pr.plan_ports(manifest, instances=5) + assert plan.effective_instances == 1 + assert plan.degraded_reason == "fixed_port" + # Honored exactly, in every world index — there is only ever world 0 once degraded, but the + # allocator itself must not silently fall back to the formula for any index it is asked for. + assert plan.port_for("tools-api", 0) == 8081 + assert plan.port_for("tools-api", 3) == 8081 + + +def test_no_fixed_port_leaves_parallelism_undegraded() -> None: + plan = pr.plan_ports(_manifest(), instances=5) + assert plan.effective_instances == 5 + assert plan.degraded_reason is None + + +def test_port_band_boundary_ordinal_99_world_7_equals_15799() -> None: + """T6, p5-round1-review: the tiling is exact with zero slack (ordinal in [0,99], world in + [0,7], per §2e item 7's caps) — nothing exercised the actual boundary before.""" + plan = pr.PortPlan( + ordinals={"last": 99}, job_shared=frozenset(), fixed_ports={}, effective_instances=8, + degraded_reason=None, + ) + assert plan.port_for("last", 7) == 15799 + + +def test_job_shared_and_per_world_port_bands_are_disjoint() -> None: + """T6: 900 ports of slack between [14000,14099] (job-shared) and [15000,15799] (per-world) — + no ordinal/world combination within the contract's own caps can alias across bands.""" + plan = pr.PortPlan( + ordinals={"shared": 99, "world": 99}, job_shared=frozenset({"shared"}), + fixed_ports={}, effective_instances=8, degraded_reason=None, + ) + job_shared_ports = {plan.port_for("shared", w) for w in range(8)} + per_world_ports = {plan.port_for("world", w) for w in range(8)} + assert job_shared_ports == {14099} # ordinal 99, the same port in every world + assert per_world_ports == {15099, 15199, 15299, 15399, 15499, 15599, 15699, 15799} + assert job_shared_ports.isdisjoint(per_world_ports) + + +# --- §2b placeholder renderer -------------------------------------------------------------------- + + +def test_configuration_name_placeholder_renders_the_capabilitys_address(tmp_path: Path) -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + endpoints = pr.build_endpoints(manifest, world_index=1, port_plan=plan, credentials=credentials) + addresses = pr.configuration_addresses_from_endpoints(endpoints) + agent = manifest.processes[2] + rendered = pr.render_environment( + agent, world_index=1, world_dir=tmp_path / "worlds" / "w1" / "agent", + port_plan=plan, configuration_addresses=addresses, + ) + assert rendered["DATABASE_URL"] == "postgresql://harness:PW@localhost:14000/w1" + assert rendered["TOOLS_API_URL"] == "http://localhost:15101" + assert rendered["NAME"] == "agent-w1" + + +def test_world_dir_db_name_host_and_port_placeholders(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], + { + **body["processes"][1], + "environment": { + **body["processes"][1]["environment"], + "SCRATCH": "{{WORLD_DIR}}", + "DB": "{{DB_NAME}}", + "PEER": "{{HOST_postgres}}:{{PORT_postgres}}", + }, + }, + body["processes"][2], + ], + } + ) + plan = pr.plan_ports(manifest, instances=1) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + endpoints = pr.build_endpoints(manifest, world_index=0, port_plan=plan, credentials=credentials) + addresses = pr.configuration_addresses_from_endpoints(endpoints) + world_dir = tmp_path / "worlds" / "w0" / "tools-api" + tools_api = manifest.processes[1] + rendered = pr.render_environment( + tools_api, world_index=0, world_dir=world_dir, port_plan=plan, + configuration_addresses=addresses, + ) + assert rendered["SCRATCH"] == str(world_dir) + assert rendered["DB"] == "w0" + assert rendered["PEER"] == "localhost:14000" + + +def test_world_index_placeholder_renders_the_bare_integer() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + rendered = pr.render_template( + "world-{{WORLD_INDEX}}", process_name="agent", world_index=3, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert rendered == "world-3" + + +def test_an_unresolvable_token_is_an_internal_error_not_a_preflight_code() -> None: + """`preflight_bundle` already validated every token against the closed vocabulary before this + ever runs — a token this function cannot resolve is a bug, not a bundle defect, so it is NOT + one of `PreflightError`'s §2e codes.""" + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_template( + "{{NOT_A_REAL_TOKEN}}", process_name="agent", world_index=0, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert excinfo.value.code == "internal_unknown_placeholder" + assert excinfo.value.stage == "render" + + +def test_a_port_placeholder_naming_an_unknown_process_is_also_an_internal_error() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_template( + "{{PORT_ghost}}", process_name="agent", world_index=0, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert excinfo.value.code == "internal_unknown_placeholder" + + +def test_build_environment_is_merged_raw_and_never_templated(tmp_path: Path) -> None: + """§2b: `build_environment` "takes NO placeholders at all" — proven end to end through + `spawn_source_process`'s env, since `render_environment`/`render_template` are never even + called on it.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process(build_environment={"TMPDIR": "{{WORLD_DIR}}"}) + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["env"] = env + return FakeHandle() + + plan = _solo_port_plan("svc") + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert captured["env"]["TMPDIR"] == "{{WORLD_DIR}}" + + +def test_render_capability_address_raises_for_an_unsupported_protocol() -> None: + """F10, p5-round1-review: a `mongodb`/`s3`/`kafka`/... capability has no defined address + shape at this seam (§3 names exactly two worked examples) — this used to silently render a + bare `://host:port`, which is not a working address for any of them.""" + capability = CapabilityV2( + protocol="mongodb", service="svc", configuration_name="MONGO_URL" + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_capability_address(capability, port=15005, world_index=0, credentials=None) + assert excinfo.value.code == "unsupported_capability_protocol" + + +def test_render_capability_address_raises_for_missing_postgres_credentials() -> None: + """S11, p5-round1-review: a typed raise for this precondition, not a bare `assert` (stripped + under `python -O`) — the same defect class as P4's N8(2).""" + capability = CapabilityV2( + protocol="postgres", service="postgres", configuration_name="DATABASE_URL" + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_capability_address(capability, port=14000, world_index=0, credentials=None) + assert excinfo.value.code == "internal_missing_credentials" + + +# --- secrets: purpose filtering (F13) ----------------------------------------------------------- + + +def test_select_process_secrets_matches_only_the_claimed_purpose() -> None: + process = _source_process(secret_purposes=["target_provider"]) + selected = pr.select_process_secrets( + process, + secret_values={"A": "1", "B": "2"}, + secret_purposes={"A": "target_provider", "B": "source_checkout"}, + ) + assert selected == {"A": "1"} + + +def test_select_process_secrets_hard_excludes_source_checkout_even_if_claimed() -> None: + """F13, p5-round1-review: §1 states `source_checkout` is gateway-only and never uploaded to + the guest — preflight's `secret_unclaimed`/`secret_missing` pair is scoped to + `target_provider` only, so a process CAN legally claim `source_checkout` too. This function + must not depend on the gateway alone to keep that promise.""" + process = _source_process(secret_purposes=["target_provider", "source_checkout"]) + selected = pr.select_process_secrets( + process, + secret_values={"A": "target-provider-value", "B": "checkout-value-must-not-move"}, + secret_purposes={"A": "target_provider", "B": "source_checkout"}, + ) + assert selected == {"A": "target-provider-value"} + + +# --- env construction (F12/F14) ----------------------------------------------------------------- + + +def test_allowlisted_ambient_env_keeps_only_the_fixed_set() -> None: + source = { + "PATH": "/bin", "HOME": "/root", "LANG": "C", "TZ": "UTC", "TMPDIR": "/tmp", + "LC_ALL": "C", "SECRET_TOKEN": "leak-me", "AWS_SECRET_ACCESS_KEY": "leak-me-too", + } + env = pr._allowlisted_ambient_env(source) + assert env == { + "PATH": "/bin", "HOME": "/root", "LANG": "C", "TZ": "UTC", "TMPDIR": "/tmp", "LC_ALL": "C", + } + + +def test_spawn_source_process_env_does_not_inherit_an_arbitrary_ambient_var( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """F12, p5-round1-review: §2b enumerates exactly what a process receives — the ambient + `svc-control` environment (a future bearer token, a `FUTUREAGI_*` marker) is not on that + list.""" + monkeypatch.setenv("FUTUREAGI_PROVISIONER_MARKER", "leak-me") + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["env"] = env + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert "FUTUREAGI_PROVISIONER_MARKER" not in captured["env"] + + +def test_base_process_env_path_prepend_has_no_empty_trailing_element(tmp_path: Path) -> None: + """F14, p5-round1-review: an unset/empty inherited `PATH` used to leave a trailing `:`, and + POSIX `execvp` reads an empty PATH element as "current directory" — cwd for build/run is the + customer's own tree, so a repo shipping an executable literally named `ls`/`git`/`sh` could + shadow the real one for any bare-name argv[0].""" + env = pr._base_process_env(tmp_path / "build" / "svc", base={}) + assert "" not in env["PATH"].split(":") + + +# --- §2b copy-based build trees ------------------------------------------------------------------- + + +def test_build_process_tree_copies_the_working_directory(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + (tmp_path / "source" / "svc" / "main.py").write_text("print(1)\n") + (tmp_path / "source" / "svc" / "sub").mkdir() + (tmp_path / "source" / "svc" / "sub" / "helper.py").write_text("print(2)\n") + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert build_dir == tmp_path / "build" / "svc" + assert (build_dir / "main.py").read_text() == "print(1)\n" + assert (build_dir / "sub" / "helper.py").read_text() == "print(2)\n" + + +def test_build_commands_get_the_venv_and_node_modules_path_prepend(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + calls: list[list[str]] = [] + + def fake_run(step, *, cwd, env, **kwargs): + calls.append(env["PATH"].split(":")[:2]) + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["npm", "ci"], ["npm", "build"]] + ) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + expected = [str(build_dir / ".venv" / "bin"), str(build_dir / "node_modules" / ".bin")] + assert len(calls) == 2 # every step, not just the first + assert calls[0] == expected + assert calls[1] == expected + + +def test_build_environment_is_merged_into_the_build_step_env(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + captured: dict[str, Any] = {} + + def fake_run(step, *, cwd, env, **kwargs): + captured["FOO"] = env.get("FOO") + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["true"]], build_environment={"FOO": "bar"} + ) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert captured["FOO"] == "bar" + + +def test_build_runs_every_step_once_per_call_in_order(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + order: list[list[str]] = [] + + def fake_run(step, *, cwd, env, **kwargs): + order.append(step) + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["step-a"], ["step-b"], ["step-c"]] + ) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert order == [["step-a"], ["step-b"], ["step-c"]] + + +def test_build_process_trees_builds_each_source_process_exactly_once(tmp_path: Path) -> None: + """No world loop exists inside `build_process_trees` at all — structural proof of "once per + job," not "once per world.\"""" + (tmp_path / "source" / "services" / "tools-api").mkdir(parents=True) + (tmp_path / "source" / ".").mkdir(exist_ok=True) + manifest = _manifest() + calls: list[str] = [] + + def fake_run(step, *, cwd, env, **kwargs): + calls.append(str(cwd)) + return subprocess.CompletedProcess(step, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=pr.plan_ports(manifest, instances=4), credentials={}, + secret_values={}, secret_purposes={}, sync_run=fake_run, + ) + build_dirs = pr.build_process_trees(manifest, source_root=tmp_path / "source", context=context) + assert set(build_dirs) == {"tools-api", "agent"} # every `source` process, no managed ones + # tools-api has 1 build step, agent has 1 build step — exactly 2 calls total, never per-world. + assert len(calls) == 2 + + +def test_a_nonzero_build_step_is_build_failed(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + return subprocess.CompletedProcess(step, 1, stdout="", stderr="npm ERR! boom") + + process = _source_process(working_directory="svc", build_commands=[["npm", "ci"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.stage == "build" + assert excinfo.value.code == "build_failed" + assert "boom" in str(excinfo.value) + + +def test_a_missing_interpreter_is_runtime_unsupported(tmp_path: Path) -> None: + """§0 (v1.8): "a repo needing an interpreter the snapshot lacks fails at BUILD time... the + build step's failure is reported `runtime_unsupported`.\"""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise FileNotFoundError(f"no such file: {step[0]!r}") + + process = _source_process( + working_directory="svc", build_commands=[["python3.13", "-m", "venv", ".venv"]] + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.stage == "build" + assert excinfo.value.code == "runtime_unsupported" + + +def test_a_missing_non_interpreter_build_tool_is_build_failed_not_runtime_unsupported( + tmp_path: Path, +) -> None: + """The missing-interpreter detection is scoped to python*/node* argv[0] patterns (§0 only + promises those two families) — a missing custom build tool is a `build_failed`, not a + `runtime_unsupported`, since the snapshot never promised it in the first place.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise FileNotFoundError(f"no such file: {step[0]!r}") + + process = _source_process(working_directory="svc", build_commands=[["some-custom-tool", "x"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.code == "build_failed" + + +def test_a_build_step_that_times_out_is_build_failed(tmp_path: Path) -> None: + """F15, p5-round1-review: an install step wedged on a private registry with no DNS answer + used to block the provisioner forever.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise subprocess.TimeoutExpired(cmd=step, timeout=kwargs.get("timeout", 0)) + + process = _source_process(working_directory="svc", build_commands=[["npm", "ci"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run, + build_step_timeout_seconds=5, + ) + assert excinfo.value.code == "build_failed" + + +# --- F5: typed copy-phase failures ---------------------------------------------------------------- + + +def test_build_process_tree_missing_working_directory_is_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source").mkdir() + process = _source_process(working_directory="does-not-exist", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + assert excinfo.value.stage == "build" + + +def test_build_process_tree_working_directory_naming_a_file_is_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source").mkdir() + (tmp_path / "source" / "svc").write_text("not a directory") + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + + +def test_build_process_tree_wraps_a_copy_failure_as_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + + def failing_copy(src: Path, dst: Path) -> None: + raise PermissionError("no") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + copy=failing_copy, + ) + assert excinfo.value.code == "source_tree_unavailable" + + +# --- F2 (BLOCKER): symlink safety, exercised against a real filesystem --------------------------- + + +def test_build_process_tree_rejects_a_symlink_escaping_the_source_root(tmp_path: Path) -> None: + """F2, p5-round1-review — BLOCKER. A repo can ship a symlink pointing outside `/work/source` + (e.g. at `/run/futureagi/capabilities.json`); `copytree`'s default (`symlinks=False`) used to + dereference it and copy the TARGET's bytes into the customer's own build tree, read as + svc-control. Verified against a REAL symlink and a REAL filesystem, not a fake copier — this + is exactly the class of bug a fake would hide.""" + outside_secret = tmp_path / "outside_secret.txt" + outside_secret.write_text("TOP SECRET") + svc_dir = tmp_path / "source" / "svc" + svc_dir.mkdir(parents=True) + (svc_dir / "evil_link").symlink_to(outside_secret) + + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + # Rejected BEFORE the copy, not after — nothing was ever materialized. + assert not (tmp_path / "build" / "svc").exists() + + +def test_build_process_tree_preserves_a_within_tree_symlink_as_a_symlink(tmp_path: Path) -> None: + """A link that stays inside the tree is legitimate and must keep working — `symlinks=True` + copies it AS a link, it does not forbid it.""" + svc_dir = tmp_path / "source" / "svc" + svc_dir.mkdir(parents=True) + (svc_dir / "real.txt").write_text("hello") + (svc_dir / "link.txt").symlink_to(svc_dir / "real.txt") + + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert (build_dir / "link.txt").is_symlink() + assert (build_dir / "link.txt").read_text() == "hello" + + +def test_build_process_tree_rejects_a_symlinked_working_directory_path_component( + tmp_path: Path, +) -> None: + """F2: the resolve-then-`is_relative_to` check catches a symlinked PATH COMPONENT too, not + just a symlinked leaf file — `working_directory: "services/tools-api"` passes the model + layer's `_safe_relative` (no `..`, not absolute) even when `/work/source/services` is ITSELF + a symlink pointing outside the checkout.""" + source_root = tmp_path / "source" + source_root.mkdir() + (source_root / "services").symlink_to(tmp_path) # escapes source_root entirely + + process = _source_process(working_directory="services/tools-api", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree(process, source_root=source_root, build_root=tmp_path / "build") + assert excinfo.value.code == "source_tree_unavailable" + + +# --- F3 (BLOCKER): path-containment defense in depth ---------------------------------------------- + + +def test_build_tree_dir_rejects_a_name_that_escapes_the_work_directory(tmp_path: Path) -> None: + """F3, p5-round1-review — BLOCKER. Defense in depth, independent of the model-layer regex + (`bundle_v2.SourceProcess`/`ManagedProcess.name`'s pattern) — these helpers take a plain + `str`, so a caller bypassing model validation still cannot walk a name-derived directory + outside `work_directory`. `pathlib` makes an absolute name a one-field catastrophe otherwise: + `Path("/work/build") / "/etc"` IS `Path("/etc")`, which the old code then `rmtree`'d.""" + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_tree_dir(tmp_path, "/etc") + assert excinfo.value.code == "process_name_invalid" + + +def test_world_scratch_dir_rejects_a_traversal_name_that_truly_escapes(tmp_path: Path) -> None: + # `world_scratch_dir` prepends TWO fixed levels ("worlds", "w") before the name — 3 levels + # of ".." is needed to escape `work_directory` itself (2 would only cancel the two prepended + # levels and land back inside it, which is not an escape). + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.world_scratch_dir(tmp_path, 0, "../../../etc") + assert excinfo.value.code == "process_name_invalid" + + +def test_managed_engine_data_dir_rejects_a_traversal_name_in_both_branches(tmp_path: Path) -> None: + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.managed_engine_data_dir(tmp_path, "/etc", world_index=None) + assert excinfo.value.code == "process_name_invalid" + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.managed_engine_data_dir(tmp_path, "../../../etc", world_index=0) + assert excinfo.value.code == "process_name_invalid" + + +def test_legitimate_process_names_are_unaffected_by_the_containment_check(tmp_path: Path) -> None: + assert pr.build_tree_dir(tmp_path, "tools-api") == tmp_path / "build" / "tools-api" + assert pr.world_scratch_dir(tmp_path, 2, "agent") == tmp_path / "worlds" / "w2" / "agent" + assert pr.managed_engine_data_dir(tmp_path, "postgres", world_index=None) == ( + tmp_path / "managed" / "postgres" + ) + + +# --- F1 (BLOCKER): `user` honored — chown + privilege drop, verified structurally ----------------- + + +def test_build_process_tree_chowns_the_copied_tree_to_the_resolved_user(tmp_path: Path) -> None: + """F1, p5-round1-review — BLOCKER. The build tree (root AND every copied file, not just the + directory) is chowned to the process's declared `user` after copy. `os.chown` to a foreign uid + needs root, which this lane does not have and must not require — `chown` is faked here and the + calls it WOULD make are asserted; see the module docstring.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + (tmp_path / "source" / "svc" / "main.py").write_text("print(1)\n") + process = _source_process(working_directory="svc", build_commands=[]) + chowned: list[tuple[str, int, int]] = [] + + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + build_dir = tmp_path / "build" / "svc" + assert (str(build_dir), 1234, 5678) in chowned + assert (str(build_dir / "main.py"), 1234, 5678) in chowned + + +def test_build_commands_run_under_the_resolved_user(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + captured: dict[str, Any] = {} + + def fake_run(step, *, cwd, env, **kwargs): + captured["user"] = kwargs.get("user") + captured["group"] = kwargs.get("group") + return subprocess.CompletedProcess(step, 0) + + process = _source_process(working_directory="svc", build_commands=[["true"]]) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run, + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + # `chown` faked too — real `os.chown` to a fake uid needs root, which this lane does not + # have; the chown call itself is asserted separately (`test_build_process_tree_chowns_...`). + chown=lambda path, uid, gid: None, + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + + +def test_build_process_tree_falls_back_unprivileged_when_user_is_not_resolvable( + tmp_path: Path, +) -> None: + """The local test lane's own shape: no `svc-*` accounts on a dev box. + `require_declared_user=False` (the default) must NOT raise — it runs unprivileged instead.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=lambda name: None, + ) + assert build_dir.is_dir() # did not raise + + +def test_build_process_tree_raises_when_user_is_required_but_not_resolvable( + tmp_path: Path, +) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=lambda name: None, require_declared_user=True, + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_source_process_applies_the_resolved_user_to_the_runner(tmp_path: Path) -> None: + """Tests: assert the resolved user IS applied — the fake runner records `user=`.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + chowned: list[tuple[str, int, int]] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["user"] = user + captured["group"] = group + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + # `{{WORLD_DIR}}` is chowned too — otherwise a process running as anyone but svc-control could + # not write into its own per-world scratch directory at all. + assert (str(world_dir), 1234, 5678) in chowned + + +def test_spawn_source_process_falls_back_unprivileged_when_user_is_not_resolvable( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["user"] = user + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=lambda name: None, + ) + assert captured["user"] is None + + +def test_spawn_source_process_raises_when_user_is_required_but_not_resolvable( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=lambda name: None, require_declared_user=True, + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_managed_process_chowns_data_dir_to_the_resolved_user(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + chowned: list[tuple[str, int, int]] = [] + + def fake_sync_run(argv, **kwargs): + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert (str(data_dir), 2222, 3333) in chowned + # The pwfile is chowned to the same user too — `initdb` runs as that user (below) and must be + # able to read its own 0600 file. + assert any(uid == 2222 and gid == 3333 and path.endswith(".pwfile") for path, uid, gid in chowned) + + +def test_default_process_runner_forwards_user_and_group_to_popen( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """F1: `subprocess.Popen(user=, group=)` requires root/CAP_SETUID to actually switch identity + — this lane does not have it and must not require it. Verified structurally: `Popen` itself is + faked, and the kwargs it would have been called with are asserted.""" + captured: dict[str, Any] = {} + + class FakePopen: + def __init__(self, *args: Any, **kwargs: Any) -> None: + captured.update(kwargs) + + def poll(self) -> None: + return None + + monkeypatch.setattr(pr.subprocess, "Popen", FakePopen) + chowned: list[tuple[str, int, int]] = [] + pr.default_process_runner( + ["true"], cwd=tmp_path, env={}, log_path=tmp_path / "x.log", user=1234, group=5678, + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + # The log file the harness creates (before the child's privilege drop) is chowned to match — + # otherwise nothing running as the child's own user could ever open it fresh afterward. + assert chowned == [(str(tmp_path / "x.log"), 1234, 5678)] + + +def test_default_process_runner_does_not_chown_the_log_when_no_user_is_given( + tmp_path: Path, +) -> None: + chowned: list[Any] = [] + handle = pr.default_process_runner( + ["python3", "-c", "pass"], cwd=tmp_path, env={"PATH": "/usr/bin:/bin"}, + log_path=tmp_path / "logs" / "proc.log", + chown=lambda *args: chowned.append(args), + ) + _reap(handle) + assert chowned == [] + + +# --- process spawn: real subprocesses via `python3 -c`, secret injection ------------------------ + + +def test_default_process_runner_spawns_a_real_subprocess_and_captures_output( + tmp_path: Path, +) -> None: + log_path = tmp_path / "logs" / "proc.log" + handle = pr.default_process_runner( + ["python3", "-c", "print('hello-from-child')"], cwd=tmp_path, env={"PATH": "/usr/bin:/bin"}, + log_path=log_path, + ) + _reap(handle) + assert "hello-from-child" in handle.captured_output() + + +def test_a_process_claiming_the_purpose_receives_its_secret(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "agent" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "agent" + script = ( + "import os,sys; sys.stdout.write('SECRET=' + os.environ.get('LIVEKIT_API_KEY', ''))" + ) + process = _source_process( + run_command=["python3", "-c", script], secret_purposes=["target_provider"] + ) + plan = _solo_port_plan("svc") + handle = pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={"LIVEKIT_API_KEY": "abc123", "OTHER": "zzz"}, + secret_purposes={"LIVEKIT_API_KEY": "target_provider", "OTHER": "source_checkout"}, + ) + _reap(handle.handle) + assert handle.handle.captured_output() == "SECRET=abc123" + + +def test_a_process_not_claiming_the_purpose_does_not_receive_the_secret(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + script = ( + "import os,sys; sys.stdout.write('SECRET=' + os.environ.get('LIVEKIT_API_KEY', ''))" + ) + process = _source_process(run_command=["python3", "-c", script], secret_purposes=[]) + plan = _solo_port_plan("svc") + handle = pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={"LIVEKIT_API_KEY": "abc123"}, + secret_purposes={"LIVEKIT_API_KEY": "target_provider"}, + ) + _reap(handle.handle) + assert handle.handle.captured_output() == "SECRET=" + + +def test_spawn_source_process_creates_the_per_world_scratch_directory(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w2" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=2, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert world_dir.is_dir() + + +def test_spawn_source_process_cwd_is_the_build_tree_not_the_world_scratch_dir( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["cwd"] = cwd + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert captured["cwd"] == build_dir + + +def test_spawn_managed_process_requires_credentials_for_postgres(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_managed_process( + postgres, port=14000, data_dir=tmp_path / "pg", credentials=None, + runner=lambda *a, **k: FakeHandle(), + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_managed_process_bootstraps_postgres_once_via_sync_run(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + bootstrap_calls: list[list[str]] = [] + run_calls: list[list[str]] = [] + + def fake_sync_run(argv, **kwargs): + bootstrap_calls.append(argv) + (data_dir).mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + run_calls.append(argv) + return FakeHandle() + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, runner=fake_runner, + sync_run=fake_sync_run, + ) + assert len(bootstrap_calls) == 1 + assert bootstrap_calls[0][0] == "initdb" + assert run_calls[0][0] == "postgres" + # No pwfile left behind after bootstrap. + assert not any(p.name.endswith(".pwfile") for p in data_dir.parent.glob(".*")) + + # A second spawn against the same already-initialized data_dir must not bootstrap again. + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, runner=fake_runner, + sync_run=fake_sync_run, + ) + assert len(bootstrap_calls) == 1 + + +# --- F7 (MAJOR): the pwfile is created 0600 atomically, not write-then-chmod --------------------- + + +def test_spawn_managed_process_creates_the_pwfile_with_o_excl_and_0600_atomically( + tmp_path: Path, +) -> None: + """F7, p5-round1-review. `write_text` then `os.chmod` creates the file at `0666 & ~umask` + (typically 0644) and only narrows it afterward — a classic create-then-chmod TOCTOU that + leaves the generated superuser password world-readable for the window in between. + `os.open`'s own arguments are captured directly (the mode at CREATION, not observed after the + fact, which cannot distinguish "created narrow" from "created wide, narrowed a moment later").""" + import os as os_module + + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + data_dir = tmp_path / "pg" + captured: dict[str, Any] = {} + real_open = os_module.open + + def spy_open(path, flags, mode=0o777, *args, **kwargs): + if str(path).endswith(".pwfile"): + captured["flags"] = flags + captured["mode"] = mode + return real_open(path, flags, mode, *args, **kwargs) + + def fake_sync_run(argv, **kwargs): + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + pr.os.open = spy_open + try: + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=fake_sync_run, + ) + finally: + pr.os.open = real_open + + assert captured["mode"] == 0o600 + assert captured["flags"] & os_module.O_EXCL + assert captured["flags"] & os_module.O_CREAT + + +# --- §2b depends_on wait ----------------------------------------------------------------------- + + +def test_depends_on_with_neither_readiness_nor_started_check_returns_immediately() -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("postgres", FakeHandle(), 14000, None) + calls = {"n": 0} + + def fake_prober(**kwargs): + calls["n"] += 1 + return True + + pr.wait_for_dependency( + manifest, "postgres", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + ) + assert calls["n"] == 0 # no readiness declared for `database` in this manifest -> immediate. + + +def test_depends_on_waits_for_the_capabilitys_readiness_probe() -> None: + manifest = _manifest() # `tools` capability has a declared readiness probe. + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + seen: list[tuple[str, int]] = [] + calls = {"n": 0} + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + calls["n"] += 1 + seen.append((host, port)) + return calls["n"] >= 3 + + ticks = {"t": 0.0} + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert calls["n"] == 3 + assert seen[0] == ("localhost", 15001) + + +def test_wait_for_dependency_requires_every_readiness_probe_backing_the_process() -> None: + """F8, p5-round1-review: a process backing two capabilities is only "ready" once ALL of its + declared readiness probes pass — matching `healthy()`'s own all-probes semantics (§4 point 3). + A process used to be treated as ready by `depends_on` the moment the FIRST-listed probe + passed, then immediately reported unhealthy by `healthy()` the instant the second had not.""" + manifest = _manifest( + lambda body: { + **body, + "capabilities": { + **body["capabilities"], + "tools_admin": { + "protocol": "http", "service": "tools-api", "configuration_name": None, + }, + }, + "readiness": [ + *body["readiness"], + { + "capability": "tools_admin", "path": "/admin/health", "timeout_seconds": 5, + "interval_seconds": 0.1, + }, + ], + } + ) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + ready = {"/health": False, "/admin/health": False} + seen_paths: list[str | None] = [] + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen_paths.append(path) + return ready[path] + + ticks = {"t": 0.0} + + def sleep(seconds: float) -> None: + ticks["t"] += seconds + if ticks["t"] >= 0.2: + ready["/health"] = True + if ticks["t"] >= 0.4: + ready["/admin/health"] = True + + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + clock=lambda: ticks["t"], sleep=sleep, + ) + assert "/admin/health" in seen_paths + assert ready["/health"] and ready["/admin/health"] + + +def test_depends_on_readiness_probe_timeout_is_a_typed_error() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + prober=lambda **kwargs: False, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.stage == "depends_on" + assert excinfo.value.code == "depends_on_timeout" + + +def test_started_check_log_marker_variant_waits_for_the_marker() -> None: + """T2, p5-round1-review: the marker now APPEARS after N polls (a mutating fake, same idiom as + `test_depends_on_waits_for_the_capabilitys_readiness_probe`'s `calls['n'] >= 3`) rather than + being seeded into `captured_output()` up front — the old fixture could not distinguish "polls + until the marker appears" from "checks once and returns.\"""" + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], + "started_check": {"log_marker": "listening", "timeout_seconds": 5}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + handle = FakeHandle("booting...\n") + spawned = pr.SpawnedWorldProcess("tools-api", handle, 15001, 0) + calls = {"n": 0} + + def mutating_captured_output() -> str: + calls["n"] += 1 + if calls["n"] >= 3: + handle._output = "booting...\nlistening on :9\n" + return handle._output + + handle.captured_output = mutating_captured_output + ticks = {"t": 0.0} + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert calls["n"] >= 3 + + +def test_started_check_log_marker_timeout_is_a_typed_error() -> None: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], + "started_check": {"log_marker": "listening", "timeout_seconds": 1}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + handle = FakeHandle("booting...\n") # marker never appears + spawned = pr.SpawnedWorldProcess("tools-api", handle, 15001, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.code == "depends_on_timeout" + + +def test_started_check_port_variant_probes_the_dependencys_own_allocated_port() -> None: + """F4, p5-round1-review — MAJOR. §2b (v1.8): `started_check.port` only SELECTS the port-probe + variant — the dialed port is always the dependency's OWN allocated port (honoring + `fixed_port`), never a literal. Exercised against a REAL bound socket on the port + `port_plan.port_for` computes (via `fixed_port`, so the exact port is deterministic rather + than depending on the formula's value not colliding with something else already listening).""" + import socket as socket_module + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + port = server.getsockname()[1] + try: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], "fixed_port": port, + "started_check": {"port": True, "timeout_seconds": 5}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + assert plan.port_for("tools-api", 0) == port + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), port, 0) + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned + ) + finally: + server.close() + + +def test_started_check_port_variant_dials_the_world_specific_formula_port( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """F4: without `fixed_port`, the dialed port must be world-`world_index`'s OWN formula port — + proven by monkeypatching the underlying TCP probe and asserting the exact port dialed, which a + real-server-bind test cannot show as directly (it can only prove "some port worked").""" + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + {**body["processes"][1], "started_check": {"port": True, "timeout_seconds": 5}}, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=3) + expected_port = plan.port_for("tools-api", 2) + assert expected_port == 15201 # ordinal 1, world 2 — NOT the same as world 0's 15001. + dialed: list[tuple[str, int]] = [] + + def fake_tcp_probe(host: str, port: int, *, timeout: float = 0.75) -> bool: + dialed.append((host, port)) + return True + + monkeypatch.setattr(pr, "_tcp_probe", fake_tcp_probe) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), expected_port, 2) + pr.wait_for_dependency(manifest, "tools-api", world_index=2, port_plan=plan, spawned=spawned) + assert dialed == [("localhost", expected_port)] + + +def test_started_check_port_variant_timeout_when_nothing_listens() -> None: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], "fixed_port": 1, + "started_check": {"port": True, "timeout_seconds": 1}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 1, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.code == "depends_on_timeout" + + +def test_spawn_world_spawns_in_dependency_order_and_waits_between(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) # skip real readiness waits + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + spawn_order: list[str] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawn_order.append(argv[0]) + return FakeHandle() + + def fake_sync_run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=plan, credentials=credentials, secret_values={}, + secret_purposes={}, runner=fake_runner, sync_run=fake_sync_run, + ) + result = pr.spawn_world(manifest, world_index=0, context=context) + assert set(result.handles) == {"postgres", "tools-api", "agent"} + # postgres has no dependency; tools-api depends on postgres; agent depends on both. + assert spawn_order.index("postgres") < spawn_order.index("node") # tools-api's run_command[0] + assert spawn_order.index("node") < spawn_order.index("python3") # agent's run_command[0] + + +def test_spawn_world_reuses_job_shared_handles_across_worlds(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + spawned_argv0s: list[str] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawned_argv0s.append(argv[0]) + return FakeHandle() + + def fake_sync_run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=plan, credentials=credentials, secret_values={}, + secret_purposes={}, runner=fake_runner, sync_run=fake_sync_run, + ) + world0 = pr.spawn_world(manifest, world_index=0, context=context) + shared = {name: handle for name, handle in world0.handles.items() if plan.is_job_shared(name)} + assert set(shared) == {"postgres"} + spawned_argv0s.clear() + world1 = pr.spawn_world(manifest, world_index=1, context=context, shared_handles=shared) + assert "postgres" not in spawned_argv0s # not respawned for world 1 + assert world1.handles["postgres"] is shared["postgres"] # the very same handle, reused + + +# --- §4.3 healthy() ------------------------------------------------------------------------------ + + +def test_healthy_dispatches_to_the_probe_for_each_declared_readiness_entry() -> None: + manifest = _manifest() # one readiness entry, for `tools` + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + }, + ) + seen: list[tuple[str, int, str | None]] = [] + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen.append((host, port, path)) + return True + + assert pr.probe_runtime_health(manifest, runtime, prober=fake_prober) is True + assert seen == [("localhost", 15001, "/health")] + + +def test_healthy_is_false_when_a_declared_probe_fails() -> None: + manifest = _manifest() + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + }, + ) + assert pr.probe_runtime_health(manifest, runtime, prober=lambda **kwargs: False) is False + + +def test_healthy_is_false_when_a_declared_probes_capability_has_no_endpoint() -> None: + manifest = _manifest() + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, endpoints={}, + ) + assert pr.probe_runtime_health(manifest, runtime, prober=lambda **kwargs: True) is False + + +def test_probe_runtime_health_parses_postgres_credentials_out_of_the_endpoint_address() -> None: + """F9, p5-round1-review: `probe_runtime_health` only ever sees `EnvironmentRuntime`, not the + job's credential map — the rendered + `postgresql://harness:@localhost:/w` address already carries everything a real + probe needs, so it is parsed back out rather than threaded separately.""" + manifest = _manifest( + lambda body: {**body, "readiness": [ + {"capability": "database", "timeout_seconds": 5, "interval_seconds": 0.1}, + ]} + ) + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "database": pr.RuntimeEndpoint( + capability="database", protocol="postgres", + address="postgresql://harness:s3cr3t@localhost:14000/w0", + configuration_name="DATABASE_URL", + ), + }, + ) + seen: dict[str, Any] = {} + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen.update(user=user, password=password, dbname=dbname) + return True + + assert pr.probe_runtime_health(manifest, runtime, prober=fake_prober) is True + assert seen == {"user": "harness", "password": "s3cr3t", "dbname": "w0"} + + +async def _run_healthy(*args: Any, **kwargs: Any) -> bool: + return await pr.healthy(*args, **kwargs) + + +def test_healthy_transitions_follow_section_3_and_never_promote() -> None: + """F6, p5-round1-review — MAJOR (T3). §3's `state` row fixes the legal transitions: + `preparing->ready`, `ready->unhealthy`, and `unhealthy->ready` ONLY via re-provision reconcile + (§4 point 1). `healthy()` is not a reconcile, so it may only ever DEMOTE — this replaces the + old test, which asserted only `preparing->ready` then `ready->unhealthy` in sequence and never + exercised `unhealthy->?` or `stopped->?` at all, so it could not have caught the promotion bug + the old implementation actually had (`RuntimeState.READY if is_healthy else UNHEALTHY`, which + unconditionally promotes ANY prior state to READY on a passing probe).""" + import asyncio + + manifest = _manifest() + endpoints = { + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + } + + def make(state: pr.RuntimeState) -> pr.EnvironmentRuntime: + return pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=state, endpoints=endpoints, + ) + + preparing = make(pr.RuntimeState.PREPARING) + assert asyncio.run(_run_healthy(manifest, preparing, prober=lambda **kwargs: True)) is True + assert preparing.state is pr.RuntimeState.READY # preparing + healthy -> ready + + still_ready = make(pr.RuntimeState.READY) + assert asyncio.run(_run_healthy(manifest, still_ready, prober=lambda **kwargs: True)) is True + assert still_ready.state is pr.RuntimeState.READY # ready stays ready + + demoted = make(pr.RuntimeState.READY) + assert asyncio.run(_run_healthy(manifest, demoted, prober=lambda **kwargs: False)) is False + assert demoted.state is pr.RuntimeState.UNHEALTHY # ready + unhealthy probe -> demote + + stays_unhealthy = make(pr.RuntimeState.UNHEALTHY) + ok = asyncio.run(_run_healthy(manifest, stays_unhealthy, prober=lambda **kwargs: True)) + assert ok is True # the PROBE passed... + assert stays_unhealthy.state is pr.RuntimeState.UNHEALTHY # ...but state is NOT promoted + + stays_stopped = make(pr.RuntimeState.STOPPED) + asyncio.run(_run_healthy(manifest, stays_stopped, prober=lambda **kwargs: True)) + assert stays_stopped.state is pr.RuntimeState.STOPPED # only provision()/close() clear this + + +# --- default probers: real postgres/http exercise, no fakes -------------------------------------- + + +def test_default_capability_prober_falls_back_to_tcp_when_psycopg_is_absent() -> None: + """`psycopg` is not installed in this test lane (import-guarded) — the postgres branch must + fall back to a bare TCP probe rather than raising `ImportError`. + + T1, p5-round1-review: the premise itself is asserted, so if `psycopg` is ever installed in + this lane the test fails LOUDLY instead of silently exercising a real-connect code path under + the same name and passing for the wrong reason.""" + assert importlib.util.find_spec("psycopg") is None, ( + "psycopg is installed in this test lane — this test's fallback premise no longer holds" + ) + import socket as socket_module + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + port = server.getsockname()[1] + try: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.POSTGRES, host="localhost", port=port, path=None + ) is True + finally: + server.close() + + +def test_default_capability_prober_tcp_probe_is_false_when_nothing_listens() -> None: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.TCP, host="localhost", port=1, path=None + ) is False + + +def test_probe_postgres_runs_a_real_select_1_when_credentials_are_supplied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """F9, p5-round1-review — MAJOR. Previously connected as `user="postgres", dbname="postgres"` + (neither exists — the catalog's `initdb -U harness` creates only `harness`) and treated almost + any resulting `OperationalError` as "answered, therefore ready." With real credentials, this + must run an actual query — proven with a fake `psycopg` module, since no real server is + available in this lane.""" + executed: list[str] = [] + + class FakeConnection: + def execute(self, query: str) -> None: + executed.append(query) + + def close(self) -> None: + pass + + def fake_connect(**kwargs: Any) -> FakeConnection: + assert kwargs["user"] == "harness" + assert kwargs["dbname"] == "w0" + return FakeConnection() + + fake_module = types.ModuleType("psycopg") + fake_module.connect = fake_connect + fake_module.OperationalError = type("OperationalError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + assert pr._probe_postgres( + "localhost", 14000, user="harness", password="pw", dbname="w0" + ) is True + assert executed == ["SELECT 1"] + + +def test_probe_postgres_treats_a_starting_up_server_as_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The old substring-match heuristic treated `FATAL: the database system is starting up` + (postgres's OWN response while still in recovery — it binds its listen socket early) as + "answered, therefore ready." A real `SELECT 1` cannot be fooled by it.""" + fake_module = types.ModuleType("psycopg") + operational_error = type("OperationalError", (Exception,), {}) + fake_module.OperationalError = operational_error + + class FakeConnection: + def execute(self, query: str) -> None: + raise operational_error("the database system is starting up") + + def close(self) -> None: + pass + + fake_module.connect = lambda **kwargs: FakeConnection() + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + assert pr._probe_postgres( + "localhost", 14000, user="harness", password="pw", dbname="w0" + ) is False + + +def test_probe_postgres_falls_back_to_tcp_when_credentials_are_not_supplied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Even when `psycopg` IS available, no generated credentials means no real query is + possible — falls back to the same TCP-only check as the psycopg-absent case, never fabricates + a connection attempt without something to authenticate with.""" + import socket as socket_module + + def fake_connect(**kwargs: Any) -> None: + raise AssertionError("must not attempt to connect without credentials") + + fake_module = types.ModuleType("psycopg") + fake_module.connect = fake_connect + fake_module.OperationalError = Exception + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + try: + port = server.getsockname()[1] + assert pr._probe_postgres("localhost", port) is True # no user/dbname given + finally: + server.close() + + +def test_probe_http_against_a_real_server_reports_2xx_as_ready() -> None: + """T7, p5-round1-review: `_probe_http` — the default prober for the one capability the shared + manifest actually declares a readiness probe for — had zero direct coverage; every readiness + test in this file injects a fake `prober` instead.""" + import http.server + import threading + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - stdlib method name + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *args: Any) -> None: # silence stderr noise + pass + + server = http.server.HTTPServer(("localhost", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + assert pr.default_capability_prober( + protocol=CapabilityProtocol.HTTP, host="localhost", port=port, path="/health" + ) is True + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_probe_http_reports_not_ready_when_nothing_listens() -> None: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.HTTP, host="localhost", port=1, path="/health" + ) is False From 1260a38f5465c79fdda70e8703475983515cb94a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 14:52:20 +0530 Subject: [PATCH 06/20] =?UTF-8?q?feat(harness):=20outbound=20channels=20?= =?UTF-8?q?=E2=80=94=20capabilities,=20canonicalization,=20spool,=20transp?= =?UTF-8?q?ort=20clients?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guest's three reporting channels per outbound-channels v1.3 and seam contract v1.12: the capabilities loader with its closed rejection vocabulary, byte-exact canonicalization with fixed digest vectors, the crash-safe sequenced spool (fsync-first, watermark clamp, corruption degrade-not-wedge, process registry), and the events/results/artifacts transport clients with the closed HTTP error map, fence latching, deadline-bounded retries, and redaction before emit. Four cold review rounds; survivors recorded as known defects in .claude/harness-alk/reports/outbound-review-r4.md (fork-path lock pairing, refusal-log volume, stale v1.11 pin). Signed-off-by: khushalsonawat --- src/fi/alk/harness/outbound.py | 2960 ++++++++++++++++++++++++++++++++ tests/harness/test_outbound.py | 2959 +++++++++++++++++++++++++++++++ 2 files changed, 5919 insertions(+) create mode 100644 src/fi/alk/harness/outbound.py create mode 100644 tests/harness/test_outbound.py diff --git a/src/fi/alk/harness/outbound.py b/src/fi/alk/harness/outbound.py new file mode 100644 index 00000000..b0fe7d3b --- /dev/null +++ b/src/fi/alk/harness/outbound.py @@ -0,0 +1,2960 @@ +"""Outbound reporting — `outbound-channels.md` v1.3, paired with `hosted-execution-seams.md` v1.11 +(the spine). All guest -> platform traffic (events, result receipts, artifacts) is outbound HTTPS; +the platform never calls in. Two halves: + +Foundations (part 1): the platform capability declaration the gateway uploads to +`/run/futureagi/capabilities.json`, the byte-exact canonical serialization every digest in the +contract is built from, and the durable local spool (with its monotonic sequence allocator) that +emission sits behind so a killed process within a live sandbox never loses or duplicates a +record. (A killed sandbox is deleted by the gateway, spool and all -- there is no in-sandbox +restart producer in the spine today, so the cross-restart recovery machinery guards a scenario +that isn't triggerable yet, but the in-process failed-write and watermark-durability guarantees +are load-bearing from the first event.) + +Transport (part 2): a channel-neutral `Transport` protocol plus a `requests`-backed production +implementation; the closed status-code error map (`classify_response`) shared by all three channel +clients; and the clients themselves — `EventsClient` (batches spooled events, advances the spool +watermark only on confirmed delivery), `ResultsClient` (typed `ResultReceiptDraft` + delivery), and +`ArtifactsClient` (content-addressed upload + `ArtifactManifestDraft` + delivery), each sharing one +retry/backoff engine (`_perform_with_retry`) that raises on the two channel-ending outcomes +(`HostedFencedError` for 401/403, `HostedChannelFailedError` for 404 exhausted) and returns a typed +`ChannelError` for everything else a caller must log-and-continue on. + +`HostedEvent`/`HostedEventDraft` model Channel 1's wire shape (the "hosted event model" the spine's +implementation-delta list calls for — `sequence`/`attempt_id`/`attempt_number`/`stage`/`digest` — +distinct from `fi.simulate.runtime.events.CanonicalEvent`, which remains the local-SDK wire and is +untouched by this module). + +Redaction (v1.3 Channel 1; seams v1.11 §3): `redact_outbound_text` scrubs URL userinfo +(`scheme://user:pw@` -> `scheme://user:***@`) plus an adapter-supplied secret-value list, applied +inside `build_event_record`/`build_result_receipt` to every free-text field the contract names +(`log.message`, `world_unhealthy.cause`, `terminal`/receipt `failure.message`, sub_goal/evaluation +`reason`). It is NOT the full "same secret-content scan as the artifact sealer" the contract also +requires — that broader scan is a separate, sealer-side obligation this module does not implement. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import random +import re +import time +import uuid +from collections.abc import Callable, Collection, Iterator, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from threading import RLock +from typing import Annotated, Any, ClassVar, Literal, Protocol +from urllib.parse import urlparse + +try: # N30: fcntl is POSIX-only; the guest is Linux-only, but the module must still IMPORT + import fcntl +except ImportError: # pragma: no cover - non-POSIX + fcntl = None # type: ignore[assignment] + +import requests +from pydantic import ( + AfterValidator, + BaseModel, + ConfigDict, + Field, + JsonValue, + ValidationError, + field_serializer, + model_validator, +) + +from .job import FailureDomain, HarnessStage + +logger = logging.getLogger(__name__) + +CAPABILITIES_SCHEMA_VERSION = "futureagi.harness-capabilities.v1" +CAPABILITIES_PATH = "/run/futureagi/capabilities.json" +EVENT_SCHEMA_VERSION = "futureagi.harness-event.v1" +RESULT_SCHEMA_VERSION = "futureagi.harness-result.v1" +MANIFEST_SCHEMA_VERSION = "futureagi.harness-manifest.v1" + +# "Channel 1" limits (outbound-channels.md v1.3) that every producer of an event record needs +# to honor before it ever reaches a transport client. MIN-12: consumed by EventsClient.flush() +# (stamps `schema_version`, clamps the batch to EVENTS_MAX_BATCH) and by HostedEventDraft's own +# size check -- no longer just declared and unused. +EVENTS_MAX_BATCH = 100 +EVENT_PAYLOAD_MAX_BYTES = 32 * 1024 +# N7: a cumulative-bytes cap on top of EVENTS_MAX_BATCH's event-count cap. 100 events * 32KB could +# reach ~3.2MB; this keeps a proactively-built batch comfortably under a common ~1MB ingress cap +# (nginx's default) so 413 is the exception, not the steady state -- EventsClient.flush() still +# halves and retries reactively on an observed 413 regardless of this cap. +EVENTS_MAX_BATCH_BYTES = 900_000 +# §3a: uploads over this size use chunked transfer; consumed by ArtifactsClient's default +# chunk_threshold_bytes. +ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES = 64 * 1024 * 1024 +# "Sequencing"/"Flush window": the drain deadline from the cancel signal, TTL, or terminal event. +# N5/N24: this module does not compute a deadline from it -- every public client method +# (EventsClient.flush / ResultsClient.push / ArtifactsClient.upload / .push_manifest) instead +# accepts an explicit `deadline: float | None` (a `time.monotonic()` value). The adapter (P10) is +# the one process-wide owner of "when did the window start," so it is the one that turns this +# constant into the deadline value it passes in -- not this module. +FLUSH_WINDOW_SECONDS = 120 + + +# ================================================================================================= +# Canonicalization -- the byte-exact serialization every digest in the contract is built from. +# ================================================================================================= + + +class OutboundError(RuntimeError): + """Generic typed failure for this module's canonicalization/digest layer -- same `code`/ + `message` shape as `CapabilitiesError`/`OutboundSpoolError`, used where neither of those is the + right domain (canonicalization itself, and shape checks that run before any spool or + capabilities object exists).""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +def canonical_bytes(value: Any) -> bytes: + """The contract's canonical form ("Canonicalization (every digest in this file)"): + ``json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False)``, + encoded UTF-8 (v1.3 pins `allow_nan=False` explicitly). This is the ONLY place that call is + made -- every digest function in this module goes through it, so a change to the algorithm + cannot happen in only one of them. + + `allow_nan=False` changes nothing about the bytes for any value that was already valid JSON -- + NaN/Infinity are not RFC 8259, so a float that would have silently produced unparseable bytes + now fails loudly here instead of downstream at the platform's parser. + + Never re-derive an already-spooled record's bytes by calling this again on retry: a dict's key + order is stable within one process but nothing guarantees float formatting or dict construction + order is bit-identical across a restart. `OutboundSpool` hands back the literal bytes it wrote; + those are what a retry re-sends, per the contract's "serialize once, spool the bytes, re-send + verbatim; never re-serialize on retry." + """ + try: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).encode("utf-8") + except ValueError as exc: + raise OutboundError("canonical_value_not_finite", str(exc)) from exc + + +def sha256_digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def event_payload_digest(payload: dict[str, Any]) -> str: + """Event digest scope: the `payload` object alone (not the envelope around it).""" + return sha256_digest(canonical_bytes(payload)) + + +def _json_native_offense(value: Any, path: str) -> str | None: + """Walks `value` looking for the first thing `canonical_bytes` cannot represent for a reason + OTHER than NaN/Infinity (which `canonical_bytes` itself catches): a non-JSON-native Python + value (`datetime`, `Decimal`, `UUID`, ...) or a non-string dict key. Returns the offending key + path (e.g. `"call.started_at"`), or `None` if the tree is clean.""" + if value is None or isinstance(value, (bool, int, float, str)): + return None + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + return f"{path}[:{key!r}]" if path else f":{key!r}" + offense = _json_native_offense(item, f"{path}.{key}" if path else key) + if offense is not None: + return offense + return None + if isinstance(value, list): + for index, item in enumerate(value): + offense = _json_native_offense(item, f"{path}[{index}]") + if offense is not None: + return offense + return None + return path or "" + + +def whole_object_digest(obj: dict[str, Any]) -> str: + """Receipt/manifest digest scope: the whole object with the `digest` key ABSENT. + + The key is popped, never set to `None` -- the contract is explicit that "absent and null are + different bytes," so silently keeping `digest: null` in the canonicalized form would compute a + different (wrong) hash than what the platform verifies against. + + Unlike an event payload (already pydantic-validated as `dict[str, JsonValue]` before it ever + reaches `event_payload_digest`), receipts and manifests reach this function as hand-built + dicts from whatever calls it -- a bare `TypeError` from `json.dumps` on a non-JSON-native value + or a non-string key is a debugging dead end with no indication of WHERE in the object the bad + value lives. This walks the tree first and raises a typed `OutboundError` naming the offending + key path instead. + """ + core = {key: value for key, value in obj.items() if key != "digest"} + offense = _json_native_offense(core, "") + if offense is not None: + raise OutboundError("digest_value_not_json_native", f"non-JSON-native value at: {offense}") + return sha256_digest(canonical_bytes(core)) + + +_DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") + + +def is_valid_digest(value: str) -> bool: + return bool(_DIGEST_PATTERN.fullmatch(value)) + + +_RFC3339_MILLIS_PATTERN = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + + +def format_rfc3339_millis(value: datetime) -> str: + """The contract's exact timestamp wire form ("Timestamps: RFC 3339, UTC, `Z`, millisecond + precision"). Unlike `HostedEvent.emitted_at` (an envelope field the event digest scope never + covers, so its exact string form doesn't matter), receipt/manifest timestamps such as + `call.started_at` sit INSIDE the `whole_object_digest` scope -- what we hash must be byte- + identical to what we send, so those fields are plain `str` on the wire models, produced only + through this function, never through a datetime's default serialization (which pydantic would + render as e.g. `+00:00` offset and six-digit microseconds, not `Z` and milliseconds). + """ + if value.tzinfo is None: + raise ValueError("naive_datetime_not_allowed") + utc = value.astimezone(timezone.utc) + return utc.strftime("%Y-%m-%dT%H:%M:%S.") + f"{utc.microsecond // 1000:03d}Z" + + +def is_valid_rfc3339_millis(value: str) -> bool: + return bool(_RFC3339_MILLIS_PATTERN.fullmatch(value)) + + +def _require_utc_millis(value: datetime) -> datetime: + """Shared `AfterValidator` for every `datetime` field this module serializes through + `format_rfc3339_millis` (`HostedEventDraft.emitted_at`, `HostedCapabilities.expires_at`): + rejects a naive datetime outright (the contract's wire form has no naive representation), + converts any other offset to UTC, and truncates to millisecond precision so the VALUE itself -- + not just its string rendering -- matches what gets sent on the wire.""" + if value.tzinfo is None: + raise ValueError("naive_datetime_not_allowed") + utc = value.astimezone(timezone.utc) + return utc.replace(microsecond=(utc.microsecond // 1000) * 1000) + + +UtcMillisDatetime = Annotated[datetime, AfterValidator(_require_utc_millis)] + + +# N9/P3: URL userinfo -- `scheme://user:pw@host` -> `scheme://user:***@host`, matching the seams +# contract's own example (`postgresql://harness:***@...`). The username group is now OPTIONAL +# (`redis://:pw@host`, the canonical empty-username shape for Redis/RabbitMQ/Mongo, is a real +# managed-store DSN form) and so is the whole password group (`https://@host`, the standard +# way a bearer token appears in git/registry output) -- a bare userinfo token is masked outright +# rather than left verbatim on the theory that "a username alone is not a secret," which is false +# for a token. +_USERINFO_PATTERN = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*://)([^\s:/?#@]*)(:[^\s/?#]*)?@") + + +def _mask_userinfo(match: re.Match[str]) -> str: + scheme, user, password = match.group(1), match.group(2), match.group(3) + return f"{scheme}{user}:***@" if password is not None else f"{scheme}***@" + + +def redact_outbound_text(value: str, extra_secret_values: tuple[str, ...] = ()) -> str: + """Scrubs a single free-text field before it can leave the sandbox on any of the three + channels (outbound-channels.md v1.3 "Redaction (enforced before emit)"; hosted-execution- + seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Two things, applied in + order: + + 1. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above) -- a password is masked and the + username kept (matching the contract's own `postgresql://harness:***@...` example); a bare + token/username-only userinfo (no `:`) is masked outright, since that shape is how a bearer + token appears, not a username. + 2. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret + values. Always `()` today: the adapter (P10) is what will know the job's declared secrets + and pass them in -- this parameter exists now so `build_event_record`/`build_result_receipt` + never need to change shape when that wiring lands. + + NOT a general secret-content scanner -- the contract's "same secret-content scan as the + artifact sealer" is a separate, sealer-side obligation. This is the narrow subset this module + can enforce on every string field it controls without false-positiving on ordinary diagnostic + text. + """ + redacted = _USERINFO_PATTERN.sub(_mask_userinfo, value) + for secret in extra_secret_values: + if secret: + redacted = redacted.replace(secret, "***") + return redacted + + +# ================================================================================================= +# Capabilities -- `/run/futureagi/capabilities.json`, "Authentication" section. +# ================================================================================================= + + +class CapabilitiesError(RuntimeError): + """A capability declaration is missing, malformed, or fails a shape rule. + + Mirrors the `code`/`message` shape `BundleV2Error`/`PreflightError` use elsewhere in this + package. `code` is one of the nine closed values in outbound-channels.md v1.3's "Capabilities- + file rejection table" -- see `load_capabilities`, which is the sole place that maps a raw + failure onto one of them. + """ + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +class HostedEndpoints(BaseModel): + """The four outbound routes, per attempt. Shape-only checks (trailing slash, https) live here + as defense-in-depth for direct construction; `load_capabilities` runs the SAME checks earlier, + outside pydantic, so each has its own `CapabilitiesError.code` instead of collapsing into the + generic `capabilities_field_invalid` (see v1.3's rejection table).""" + + model_config = ConfigDict(extra="forbid") + + events: str + results: str + artifacts: str + scenarios: str + + @model_validator(mode="after") + def _shape(self) -> "HostedEndpoints": + for name in ("events", "results", "artifacts", "scenarios"): + value = getattr(self, name) + if not value or not value.endswith("/"): + raise ValueError(f"{name} endpoint must end with '/'") + if not value.startswith("https://"): + raise ValueError(f"{name} endpoint must use https") + return self + + +class HostedCapabilities(BaseModel): + """The per-attempt bearer plus the four endpoint URLs. Loaded once at emitter startup from the + file the gateway uploads (§0 step 4) -- see `load_capabilities`, which also implements the + contract's "loaded into memory ... and unlinked" lifetime rule.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: str + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + fence: str = Field(min_length=1) + expires_at: UtcMillisDatetime + token: str = Field(min_length=1) + endpoints: HostedEndpoints + + @model_validator(mode="after") + def _schema_shape(self) -> "HostedCapabilities": + # Defense-in-depth only -- `load_capabilities` checks this first, outside pydantic, so it + # can raise `capabilities_schema_unsupported` specifically rather than the generic + # `capabilities_field_invalid` this validator's `ValueError` would collapse into. + if self.schema_version != CAPABILITIES_SCHEMA_VERSION: + raise ValueError(f"unsupported schema_version: {self.schema_version}") + return self + + @field_serializer("expires_at") + def _serialize_expires_at(self, value: datetime) -> str: + return format_rfc3339_millis(value) + + def auth_headers(self) -> dict[str, str]: + """"Every request: `Authorization: Bearer ` + `X-Harness-Fence: `." Pure + formatting -- issuing the request itself is a P8 transport-client concern.""" + return {"Authorization": f"Bearer {self.token}", "X-Harness-Fence": self.fence} + + def event_builder(self, *, extra_secret_values: tuple[str, ...] = ()) -> Callable[..., dict[str, Any]]: + """A `build_event_record`-shaped callable with `job_id`/`attempt_id`/`attempt_number` + closed over from THIS capabilities object. The contract's `403 attempt_mismatch` fires when + an event's identity disagrees with the token authenticating it -- binding these three + fields here makes that class of caller bug unrepresentable at the call site instead of a + runtime 403 discovered mid-attempt. + + P4: `extra_secret_values` is bound here too, alongside identity, rather than left as a + per-call parameter -- `build_event_record` could not previously receive the job's declared + secret list at all through this binder, so a caller wired for `event_builder()` had no way + to satisfy N9's redaction requirement on `log.message`/`terminal.failure.message` without + routing around this method entirely. Binding once here matches how identity is already + bound and gives the adapter one place to get it wrong instead of two. + """ + + def build( + *, + event_id: str, + emitted_at: datetime, + stage: HarnessStage, + type: OutboundEventType, + payload: dict[str, JsonValue], + ) -> dict[str, Any]: + return build_event_record( + event_id=event_id, + job_id=self.job_id, + attempt_id=self.attempt_id, + attempt_number=self.attempt_number, + emitted_at=emitted_at, + stage=stage, + type=type, + payload=payload, + extra_secret_values=extra_secret_values, + ) + + return build + + +def _endpoint_matches_attempt(url: str, attempt_id: str) -> bool: + """"an endpoint's `` path segment disagrees with the declared `attempt_id`" + (`capabilities_attempt_mismatch`, v1.3) -- checked as a whole path segment, not a substring, so + an attempt_id that happens to be a substring of another segment can't produce a false match.""" + segments = [segment for segment in urlparse(url).path.split("/") if segment] + return attempt_id in segments + + +def _redact_validation_error(exc: ValidationError) -> str: + """Builds a `capabilities_field_invalid` message from `loc`/`msg` only -- pydantic's default + `str(exc)` embeds each failing field's `input_value`, and the capabilities file carries the + bearer token; a caller that logs this message must never be able to leak it.""" + parts = [] + for error in exc.errors(): + loc = ".".join(str(part) for part in error.get("loc", ())) + msg = error.get("msg", "") + parts.append(f"{loc}: {msg}" if loc else msg) + return "; ".join(parts) or "capabilities file failed validation" + + +def _warn_if_capabilities_file_insecure(target: Path) -> None: + """"owner svc-control, mode 0600" is the contract's posture for this file, but a wrong mode or + owner is NOT a load-time rejection (MIN-5, fail-safe): the bearer is only a per-attempt token + that expires on its own, and refusing to load it entirely over a permissions mistake would turn + a minor hardening gap into a hard attempt failure. Loud warning only.""" + try: + info = target.stat() + except OSError: + return + mode = info.st_mode & 0o777 + if mode != 0o600: + logger.warning( + "%s: capabilities file mode is %o, expected 0600 -- a world/group-readable bearer in " + "a multi-user sandbox is a leak risk (not blocking the load)", + target, mode, + ) + try: + running_uid = os.geteuid() + except AttributeError: + return # os.geteuid() is POSIX-only + if info.st_uid != running_uid: + logger.warning( + "%s: capabilities file is owned by uid %s, not the running uid %s -- expected owner " + "svc-control per the contract (not blocking the load)", + target, info.st_uid, running_uid, + ) + + +def load_capabilities( + path: str | Path = CAPABILITIES_PATH, + *, + unlink: bool = True, + now: Callable[[], datetime] | None = None, + on_unlink_failure: Callable[[OSError], None] | None = None, +) -> HostedCapabilities: + """Parse and validate one capabilities file against outbound-channels.md v1.3's closed + "Capabilities-file rejection table" (nine codes, all reachable as `CapabilitiesError.code`). + + A guest that cannot load this file has no channel at all -- no token, no endpoints -- so it + cannot report its own failure; every branch below raises before any network-capable object + exists. Most checks run BEFORE `HostedCapabilities.model_validate`, outside any pydantic + validator: wrapping them in a pydantic `ValidationError` (the old shape) meant they only ever + surfaced as the generic `capabilities_field_invalid`, never as their own named code -- checking + here first makes each one an independently raised, independently testable `CapabilitiesError`. + + ``unlink=True`` (the default) implements "loaded into memory at emitter startup and unlinked" -- + the file is only ever removed AFTER a successful parse and validation, never before, so a + crash mid-load leaves the file in place for the next attempt to read rather than destroying the + only copy of a not-yet-consumed bearer. A failure to unlink is never fatal to an otherwise- + successful load (the sandbox is destroyed by the gateway at attempt end regardless) but is no + longer silently swallowed either: it is reported via `on_unlink_failure` if given, else logged + (MIN-7) -- the caller can still tell a 0600 bearer may be lingering on disk. + + ``now`` is injectable (defaults to the real clock) so `capabilities_expired` is testable without + manipulating the wall clock. + """ + target = Path(path).expanduser() + try: + exists = target.is_file() + except OSError as exc: + raise CapabilitiesError("capabilities_file_unreadable", str(exc)) from exc + if not exists: + raise CapabilitiesError("capabilities_file_missing", str(target)) + _warn_if_capabilities_file_insecure(target) + + try: + text = target.read_text(encoding="utf-8") + except OSError as exc: + raise CapabilitiesError("capabilities_file_unreadable", str(exc)) from exc + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + raise CapabilitiesError("capabilities_file_malformed", str(exc)) from exc + if not isinstance(raw, dict): + raise CapabilitiesError("capabilities_file_malformed", "not a JSON object") + + schema_version = raw.get("schema_version") + if schema_version != CAPABILITIES_SCHEMA_VERSION: + raise CapabilitiesError("capabilities_schema_unsupported", str(schema_version)) + + attempt_id = raw.get("attempt_id") + endpoints_raw = raw.get("endpoints") + if isinstance(endpoints_raw, dict): + for name in ("events", "results", "artifacts", "scenarios"): + value = endpoints_raw.get(name) + if not isinstance(value, str): + continue # missing/wrong-typed -- a shape error pydantic below will catch + if not value.endswith("/"): + raise CapabilitiesError("capabilities_endpoint_invalid", f"endpoints.{name} must end with '/'") + if not value.startswith("https://"): + raise CapabilitiesError("capabilities_endpoint_insecure", f"endpoints.{name} must use https") + if isinstance(attempt_id, str) and attempt_id and not _endpoint_matches_attempt(value, attempt_id): + raise CapabilitiesError( + "capabilities_attempt_mismatch", + f"endpoints.{name} does not carry the declared attempt_id {attempt_id!r}", + ) + + try: + capabilities = HostedCapabilities.model_validate(raw) + except ValidationError as exc: + raise CapabilitiesError("capabilities_field_invalid", _redact_validation_error(exc)) from exc + + current_time = (now or (lambda: datetime.now(timezone.utc)))() + if capabilities.expires_at <= current_time: + raise CapabilitiesError("capabilities_expired", f"expires_at={capabilities.expires_at.isoformat()}") + + if unlink: + try: + target.unlink() + except OSError as exc: + if on_unlink_failure is not None: + on_unlink_failure(exc) + else: + logger.warning( + "%s: failed to unlink the capabilities file after a successful load (%s) -- a " + "0600 bearer may still be on disk; the sandbox is destroyed at attempt end " + "regardless, so this does not fail the load", + target, exc, + ) + return capabilities + + +# ================================================================================================= +# Channel 1 -- Events. The closed `type` vocabulary and each type's payload shape. +# ================================================================================================= + + +class DegradeReason(str, Enum): + CONFORMANCE_GATE_FAILED = "conformance_gate_failed" + FIXED_PORT = "fixed_port" + + +class LogLevel(str, Enum): + DEBUG = "debug" + INFO = "info" + WARNING = "warning" + ERROR = "error" + + +class TerminalReason(str, Enum): + TTL_EXCEEDED = "ttl_exceeded" + USER_CANCELED = "user_canceled" + + +class OutboundEventType(str, Enum): + STAGE_CHANGED = "stage_changed" + PARALLELISM_DEGRADED = "parallelism_degraded" + BASELINE_FROZEN = "baseline_frozen" + BASELINE_INPUTS_CHANGED = "baseline_inputs_changed" + WORLD_UNHEALTHY = "world_unhealthy" + SCENARIO_STARTED = "scenario_started" + SCENARIO_RETRIED = "scenario_retried" + LOG = "log" + TERMINAL = "terminal" + + +class StageChangedPayload(BaseModel): + """No `populate_by_name` -- the wire key is `from` (a Python keyword, hence the `from_stage` + attribute name + alias), and this model is validation-only (`HostedEventDraft` never + normalizes `self.payload`; it emits the caller's dict verbatim). Allowing population by the + attribute name too would let a caller who writes `from_stage` in their payload dict pass + validation while spooling an undefined wire key -- `populate_by_name=True` previously made + exactly that mistake succeed silently.""" + + model_config = ConfigDict(extra="forbid") + + from_stage: HarnessStage | None = Field(alias="from") + to: HarnessStage + + +class ParallelismDegradedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + requested: int = Field(ge=1) + effective: int = Field(ge=1) + reason: DegradeReason + + @model_validator(mode="after") + def _range(self) -> "ParallelismDegradedPayload": + if not (1 <= self.effective < self.requested): + raise ValueError( + f"parallelism_degraded_effective_out_of_range: effective={self.effective} " + f"requested={self.requested}" + ) + return self + + +class BaselineFrozenPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + inputs_digest: str + baseline_ref: str + + +class BaselineInputsChangedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + previous_digest: str | None + current_digest: str + + +class WorldUnhealthyPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + world_index: int = Field(ge=0) + cause: str = Field(max_length=200) + + +class ScenarioStartedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + scenario_key: str = Field(min_length=1) + world_index: int = Field(ge=0) + scenario_attempt: Literal[1, 2] + + +class ScenarioRetriedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + scenario_key: str = Field(min_length=1) + from_world: int = Field(ge=0) + to_world: int = Field(ge=0) + + +class LogPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + level: LogLevel + message: str + + +_LOG_TRUNCATION_MARKER = "…[truncated]" + + +def truncate_log_message( + level: str, message: str, *, max_payload_bytes: int = EVENT_PAYLOAD_MAX_BYTES +) -> str: + """The `log` event's own contract rule -- "truncated to fit with a trailing `…[truncated]` + marker" -- unlike the other eight event types, which are hard-rejected when oversized (M8: + `log` is the contract's designated escape hatch for reporting every other permanent failure, so + the one channel meant to report an oversized diagnostic must not itself throw on size). + + Sizing is against `canonical_bytes({"level": level, "message": })`, the exact bytes + `HostedEventDraft`'s own size check measures, so a truncated message is guaranteed to fit + before it ever reaches that check. A no-op when already within budget. + """ + if len(canonical_bytes({"level": level, "message": message})) <= max_payload_bytes: + return message + if len(canonical_bytes({"level": level, "message": _LOG_TRUNCATION_MARKER})) > max_payload_bytes: + raise OutboundError( + "log_payload_budget_too_small", + f"max_payload_bytes={max_payload_bytes} cannot fit even the truncation marker", + ) + lo, hi, best = 0, len(message), "" + while lo <= hi: + mid = (lo + hi) // 2 + candidate = message[:mid] + _LOG_TRUNCATION_MARKER + if len(canonical_bytes({"level": level, "message": candidate})) <= max_payload_bytes: + best = candidate + lo = mid + 1 + else: + hi = mid - 1 + return best + + +class TerminalFailure(BaseModel): + """The terminal event's `failure` shape: `{domain, stage, code, message}` -- a leaner subset of + `job.HarnessFailure` (no `retryable`/`details`), matching §"Event `type` vocabulary" exactly so + a canonicalized terminal payload never carries fields the contract doesn't name.""" + + model_config = ConfigDict(extra="forbid") + + domain: FailureDomain + stage: HarnessStage + code: str + message: str + + +class ScenarioCounts(BaseModel): + model_config = ConfigDict(extra="forbid") + + passed: int = Field(ge=0) + failed: int = Field(ge=0) + errored: int = Field(ge=0) + skipped: int = Field(ge=0) + + +class TerminalPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + stage: HarnessStage + reason: TerminalReason | None + failure: TerminalFailure | None + scenario_counts: ScenarioCounts + + @model_validator(mode="after") + def _terminal_stage(self) -> "TerminalPayload": + if not self.stage.terminal: + raise ValueError(f"terminal_event_stage_not_terminal: {self.stage.value}") + return self + + +_PAYLOAD_MODELS: dict[OutboundEventType, type[BaseModel]] = { + OutboundEventType.STAGE_CHANGED: StageChangedPayload, + OutboundEventType.PARALLELISM_DEGRADED: ParallelismDegradedPayload, + OutboundEventType.BASELINE_FROZEN: BaselineFrozenPayload, + OutboundEventType.BASELINE_INPUTS_CHANGED: BaselineInputsChangedPayload, + OutboundEventType.WORLD_UNHEALTHY: WorldUnhealthyPayload, + OutboundEventType.SCENARIO_STARTED: ScenarioStartedPayload, + OutboundEventType.SCENARIO_RETRIED: ScenarioRetriedPayload, + OutboundEventType.LOG: LogPayload, + OutboundEventType.TERMINAL: TerminalPayload, +} + + +class HostedEventDraft(BaseModel): + """A Channel 1 event before spool-assigned `sequence`. The caller supplies `digest` itself + (computed via `event_payload_digest`) -- the model then re-derives it and rejects a mismatch, + so a caller can never accidentally spool a record whose embedded digest disagrees with its own + payload bytes. + """ + + model_config = ConfigDict(extra="forbid") + + event_id: str = Field(min_length=1, max_length=64) + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + emitted_at: UtcMillisDatetime + stage: HarnessStage + type: OutboundEventType + payload: dict[str, JsonValue] + digest: str + + @field_serializer("emitted_at") + def _serialize_emitted_at(self, value: datetime) -> str: + return format_rfc3339_millis(value) + + @model_validator(mode="after") + def _validate(self) -> "HostedEventDraft": + # `max_length=64` above counts characters; the contract says "opaque <=64 chars," but the + # platform's column is presumably bytes -- a multi-byte-UTF-8 id could pass the character + # count and still overflow it (N-2). + if len(self.event_id.encode("utf-8")) > 64: + raise ValueError(f"event_id_too_long_in_bytes: {self.event_id!r}") + if not is_valid_digest(self.digest): + raise ValueError(f"event_digest_invalid: {self.digest!r}") + expected = event_payload_digest(self.payload) + if self.digest != expected: + raise ValueError("event_digest_mismatch") + if len(canonical_bytes(self.payload)) > EVENT_PAYLOAD_MAX_BYTES: + raise ValueError(f"event_payload_too_large: {self.event_id}") + + model_cls = _PAYLOAD_MODELS[self.type] + try: + model_cls.model_validate(self.payload) + except ValidationError as exc: + raise ValueError(f"event_payload_invalid: {self.type.value}: {exc}") from exc + + if self.type is OutboundEventType.STAGE_CHANGED and self.payload.get("to") != self.stage.value: + raise ValueError("event_stage_mismatch: stage_changed.to must equal the event's stage") + if self.type is OutboundEventType.TERMINAL and self.payload.get("stage") != self.stage.value: + raise ValueError("event_stage_mismatch: terminal.stage must equal the event's stage") + return self + + +class HostedEvent(HostedEventDraft): + """The full Channel 1 wire object, `sequence` included -- what actually gets spooled and sent. + Distinct from `fi.simulate.runtime.events.CanonicalEvent` (the untouched local-SDK wire).""" + + sequence: int = Field(ge=1) + + +def build_event_record( + *, + event_id: str, + job_id: str, + attempt_id: str, + attempt_number: int, + emitted_at: datetime, + stage: HarnessStage, + type: OutboundEventType, + payload: dict[str, JsonValue], + extra_secret_values: tuple[str, ...] = (), +) -> dict[str, Any]: + """Validate one event's shape and compute its digest, returning a plain dict with no + `sequence` key -- ready for `OutboundSpool.append`, which assigns `sequence` and performs the + one-time serialization. Raises `ValueError` (via pydantic) on any shape violation; callers that + want a typed/coded failure should catch `pydantic.ValidationError` themselves, matching how the + rest of this package surfaces model-layer rejections (`bundle_v2.py`, `job.py`). + + N9: `redact_outbound_text` runs on every free-text field the contract names BEFORE the digest + is computed -- `log.message`, `world_unhealthy.cause`, `terminal.failure.{code,message}`, + `baseline_frozen.baseline_ref` (P8) -- so the embedded digest always matches the redacted bytes + actually spooled and sent, never the unredacted original. `log` events are then truncated to + fit (M8), also before the digest -- redact first, since truncation must size against the final + (redacted) text, not text that would still shrink again once secrets are scrubbed. Every other + event type still hard-rejects when oversized, via `HostedEventDraft`'s own size check. + + P8: `failure.code` is redacted alongside `failure.message` -- both are free `str` fields (the + contract's `code` vocabularies are closed in prose, but nothing enforces that here), and + `baseline_ref` is likewise a free `str` that plausibly carries an OCI/registry reference in the + same `https://@registry/...` shape `redact_outbound_text` already scrubs. + """ + payload = dict(payload) + if type is OutboundEventType.LOG: + level, message = payload.get("level"), payload.get("message") + if isinstance(message, str): + message = redact_outbound_text(message, extra_secret_values) + if isinstance(level, str): + message = truncate_log_message(level, message) + payload["message"] = message + elif type is OutboundEventType.WORLD_UNHEALTHY: + cause = payload.get("cause") + if isinstance(cause, str): + payload["cause"] = redact_outbound_text(cause, extra_secret_values) + elif type is OutboundEventType.BASELINE_FROZEN: + baseline_ref = payload.get("baseline_ref") + if isinstance(baseline_ref, str): + payload["baseline_ref"] = redact_outbound_text(baseline_ref, extra_secret_values) + elif type is OutboundEventType.TERMINAL: + failure = payload.get("failure") + if isinstance(failure, dict): + redacted_failure = dict(failure) + if isinstance(failure.get("code"), str): + redacted_failure["code"] = redact_outbound_text(failure["code"], extra_secret_values) + if isinstance(failure.get("message"), str): + redacted_failure["message"] = redact_outbound_text(failure["message"], extra_secret_values) + payload["failure"] = redacted_failure + digest = event_payload_digest(payload) + draft = HostedEventDraft( + event_id=event_id, + job_id=job_id, + attempt_id=attempt_id, + attempt_number=attempt_number, + emitted_at=emitted_at, + stage=stage, + type=type, + payload=payload, + digest=digest, + ) + return draft.model_dump(mode="json") + + +# ================================================================================================= +# Spool -- durable on-disk queue + monotonic sequence allocator. +# ================================================================================================= + + +@dataclass(frozen=True) +class SpooledRecord: + """One durably-appended record. `body` is the EXACT canonical bytes written to disk -- a P8 + transport client re-sends `body` verbatim on retry rather than re-serializing the decoded + dict, per the contract's "serialize once ... never re-serialize on retry.\"""" + + sequence: int | None + body: bytes + + def decode(self) -> dict[str, Any]: + return json.loads(self.body.decode("utf-8")) + + +class OutboundSpoolError(RuntimeError): + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +def _iter_complete_records( + data: bytes, +) -> tuple[list[tuple[int, bytes, dict[str, Any] | list[Any] | str | int | float | bool | None]], int, int | None]: + """Shared by `_recover` and `records()`/`pending_since_watermark()` -- the ONE place spool + bytes are split into records, so both ever agree on what a "complete record" is. + + Splits on the literal `b"\\n"` byte (N-1: not `bytes.splitlines()`, which also treats `\\r`/ + `\\r\\n` as separators `append` never writes -- canonical JSON never contains a raw newline of + any kind, so `\\n` is the only byte that can legitimately end a line). + + Returns `(records_before_corruption, valid_length, corruption_offset)` (N8): + - `records_before_corruption`: every complete, successfully decoded, non-blank line UP TO the + first corrupt one (or all of them, if none is corrupt), as `(start_offset, raw_line, + decoded_value)`, in file order. + - `corruption_offset`: the byte offset of the first `\\n`-terminated line that failed to parse + as JSON -- genuine corruption (a torn write, by definition, never got its trailing `\\n`, so + this is never that) -- or `None` if no such line was found. Once found, scanning STOPS: never + renumber or trust anything past a corrupt byte (B1). + - `valid_length`: when `corruption_offset is None`, the byte offset immediately after the last + complete line -- where `_recover` truncates away a torn tail. When corruption WAS found, this + equals `corruption_offset` and callers must NOT use it to truncate -- corrupt bytes are left + on disk, never deleted (N8: "never truncates mid-file damage"). + """ + records: list[tuple[int, bytes, Any]] = [] + start = 0 + while True: + newline_index = data.find(b"\n", start) + if newline_index == -1: + return records, start, None + line = data[start:newline_index] + line_end = newline_index + 1 + if line: + try: + decoded = json.loads(line.decode("utf-8")) + except ValueError: + return records, start, start + records.append((start, line, decoded)) + start = line_end + + +class OutboundSpool: + """Durable, crash-safe local queue for one outbound record stream (events, results, or + artifact-manifest state) -- the "fsync-first local spool" the contract requires emission to sit + behind ("Emission is an async flusher over the fsync-first local spool -- it never blocks the + call loop"). `sequenced=True` is for Channel 1 only ("Sequencing: one allocator, one lock, + assigned at spool append, contiguous from 1" -- receipts and the manifest carry no `sequence` + field and use their own idempotency keys instead). + + ONE ALLOCATOR PER STREAM (M6): `OutboundSpool(root, name, ...)` is keyed on + `(resolved_root, name)` -- a second construction for the same key, anywhere in this process, + returns the SAME instance rather than a second independent allocator (see `__new__`); a second + OS PROCESS pointing at the same directory fails loudly instead, via an `fcntl.flock` on + `.spool.lock` held for the life of the owning instance. + + Recovery rule (the contract specifies the sequencing invariant -- contiguous from 1, no gaps or + dupes across a restart -- but not the recovery mechanism; this is the FAIL-SAFE/REVERSIBLE + choice under the stuck-decision rule, surfaced in the P7 report): + + The next sequence number is derived by SCANNING the spool's own JSONL log at startup, never + from an independent counter file. A separate counter file could be durably advanced in a write + that lands, while the record it was allocated for does not (crash between the two writes), + producing a sequence number with no corresponding record -- a permanent, undetectable gap. + Scanning the log makes the durably-written records themselves the only source of truth. The + scan is then reconciled against the durable watermark (M5): `next_sequence = + max(max_sequence_in_log, watermark) + 1` -- the watermark can be AHEAD of the log (the log lost + already-processed records, e.g. via the directory-fsync gap M2 closes) but never behind it, so + taking the max is always safe and never skips a record that was actually spooled. + + A torn last line -- a crash mid-write, since a single `write()` of `body + b"\\n"` is not + guaranteed atomic by POSIX for a regular file -- is detected (the trailing bytes don't end in + `b"\\n"`) and the file is truncated back to the end of the last complete record before any + further append. The next append then reuses that same sequence number rather than skipping it: + a torn write is treated as though it never happened, closing the gap instead of creating one. + This depends on canonical JSON never containing a raw newline byte (control characters are + always escaped by `json.dumps`), which `append` asserts on every write. A COMPLETE line that + still fails to parse is a different, worse fault: genuine corruption degrades the stream to its + readable prefix rather than raising (N8, `is_corrupt`) -- see `_recover`/`records()`. + + Registration (N4): a construction is only added to `_registry` at the END of a successful + `__init__`, under `_registry_lock` -- never a half-built instance. A failed construction (e.g. + `mkdir` EACCES, or `_recover` finding corruption) therefore never poisons the key: it raises + without registering anything, and the NEXT `OutboundSpool(root, name, ...)` call starts a + completely fresh attempt rather than returning (or conflicting with) wreckage. The real mutual- + exclusion primitive across a same-key construction race is `_acquire_process_lock`'s `flock` + (an OS-level device, safe across threads and processes alike) -- the registry dict on top is + only a same-process memoization cache. + """ + + _registry: ClassVar[dict[tuple[Path, str], "OutboundSpool"]] = {} + _registry_lock: ClassVar[RLock] = RLock() + + def __new__(cls, root: str | Path, name: str, *, sequenced: bool) -> "OutboundSpool": + resolved_root = Path(root).expanduser().resolve() + key = (resolved_root, name) + with cls._registry_lock: + existing = cls._registry.get(key) + if existing is not None: + # getattr belt-and-braces (N4): `existing` is only ever registered after a fully + # successful __init__, so `_sequenced` should always be set -- but never trust that + # invariant harder than a defensive read costs. + if getattr(existing, "_sequenced", None) != sequenced: + raise OutboundSpoolError( + "outbound_spool_sequenced_mismatch", + f"{name}: existing instance has sequenced={getattr(existing, '_sequenced', None)}, " + f"requested sequenced={sequenced}", + ) + return existing + return super().__new__(cls) + + def __init__(self, root: str | Path, name: str, *, sequenced: bool) -> None: + if getattr(self, "_initialized", False): + return + resolved_root = Path(root).expanduser().resolve() + key = (resolved_root, name) + with type(self)._registry_lock: + if getattr(self, "_initialized", False): + return + lock_fd: int | None = None + try: + self.root = resolved_root + self.root.mkdir(parents=True, exist_ok=True) + try: + os.chmod(self.root, 0o700) # MIN-10: mkdir's mode is subject to umask + except OSError: + pass + self._name = name + self._sequenced = sequenced + self._path = self.root / f"{name}.spool.jsonl" + self._watermark_path = self.root / f"{name}.spool.watermark.json" + self._lock = RLock() + self._dir_synced = False + self._offset_by_sequence: dict[int, int] = {} + self._next_sequence = 1 if sequenced else None + self._poisoned = False # N12 + self._corrupt_since_offset: int | None = None # N8 + self._forked = False # N25 + self._closed = False # P2 + self._lock_fd = self._acquire_process_lock() + lock_fd = self._lock_fd + self._recover() + except BaseException: + # N4: never leave a half-built instance registered -- it was never added (below), + # so there is nothing to evict; just release whatever this attempt itself opened. + if lock_fd is not None: + try: + os.close(lock_fd) + except OSError: + pass + raise + self._initialized = True + type(self)._registry[key] = self + + @classmethod + def _forget_for_tests(cls, root: str | Path, name: str) -> None: + """Test-only escape hatch: a real process restart naturally gets a fresh, empty registry + (a new interpreter); simulating that WITHIN one process/test needs an explicit evict so the + next `OutboundSpool(root, name, ...)` call re-scans the on-disk log instead of returning the + still-live cached instance. Never called from production code.""" + resolved_root = Path(root).expanduser().resolve() + with cls._registry_lock: + instance = cls._registry.get((resolved_root, name)) + if instance is not None: + instance.close() + + @classmethod + def _clear_registry_for_tests(cls) -> None: + """Broader sibling of `_forget_for_tests`: releases every cached instance's lock fd and + empties the registry. Intended for an autouse test fixture so flock fds don't accumulate + across a whole test session.""" + with cls._registry_lock: + instances = list(cls._registry.values()) + for instance in instances: + instance.close() + + def close(self) -> None: + """N26/P2: releases this instance's process lock and evicts it from the registry, so a + later `OutboundSpool(root, name, ...)` call re-scans the on-disk log instead of reusing this + instance. Idempotent -- safe to call more than once, or on an instance never fully + constructed. + + P2: also sets `_closed`, so THIS instance -- not just the registry slot -- refuses further + mutation. Evicting the registry entry alone left the closed instance itself fully live: a + caller still holding a reference could keep appending with no flock held (the lock fd was + released), and a fresh `OutboundSpool(...)` call for the same key would allocate a second, + independent `_next_sequence` -- two live allocators for one stream, each unaware of the + other, which is exactly the M6 invariant `close()` must not itself reopen.""" + with type(self)._registry_lock: + key = (getattr(self, "root", None), getattr(self, "_name", None)) + if type(self)._registry.get(key) is self: + del type(self)._registry[key] + fd = getattr(self, "_lock_fd", None) + if fd is not None: + try: + os.close(fd) + except OSError: + pass + self._lock_fd = None + self._closed = True + + def _require_writable(self) -> None: + """P2/P7: the single gate `append`, `advance_watermark`, and `_rewrite_retaining` all call + before touching disk -- refuses a closed, forked, or poisoned instance instead of letting it + silently duplicate allocators, advance a parent's watermark from a forked child, or compound + a rollback failure `_truncate_to` already flagged as unrecoverable.""" + if self._closed: + raise OutboundSpoolError( + "outbound_spool_closed", + f"{self._name}: this OutboundSpool was closed; construct a new one for this stream", + ) + if self._forked: + raise OutboundSpoolError( + "outbound_spool_forked", + f"{self._name}: this OutboundSpool was constructed before a fork; construct a " + f"new one in the child process instead of reusing this one", + ) + if self._poisoned: + raise OutboundSpoolError( + "outbound_spool_poisoned", + f"{self._name}: a prior rollback failed and left this spool in an unknown " + f"state; it must not be mutated again", + ) + + def _acquire_process_lock(self) -> int: + """M6: cross-PROCESS protection (the in-process registry above only protects against a + second Python-level instance in this same interpreter). Held for the life of this instance + -- released implicitly when its fd closes (process exit, `close()`, or `_forget_for_tests` + in tests).""" + if fcntl is None: # N30 + raise OutboundSpoolError( + "outbound_spool_platform_unsupported", + f"{self._name}: fcntl (POSIX file locking) is unavailable on this platform", + ) + lock_path = self.root / f"{self._name}.spool.lock" + fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + os.close(fd) + raise OutboundSpoolError( + "outbound_spool_locked", f"{self._name}: already locked by another process" + ) from exc + return fd + + def _fsync_dir(self) -> None: + fd = os.open(str(self.root), os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def _report_corruption(self, offset: int) -> None: + """N8: the client-visible half of "degrade, don't wedge" -- `is_corrupt`/`corruption_offset` + stay true/set for the life of this instance once discovered, but the loud `logger.error` + fires only the FIRST time (repeated reads of an already-known-corrupt spool would otherwise + spam the log every flush cycle).""" + with self._lock: + already_reported = self._corrupt_since_offset is not None + self._corrupt_since_offset = offset + if not already_reported: + logger.error( + "%s: spool corrupt at byte offset %d -- the file is left untouched; only records " + "before that offset are trusted. Reading degrades to the readable prefix rather " + "than raising -- this is reported once per process, not per read.", + self._name, offset, + ) + + @property + def is_corrupt(self) -> bool: + """N8: set once `_recover` or a later read finds a genuinely corrupt (not merely torn) + record. A caller can turn this into a `log`-kind event; nothing in this class does that + itself (no scenario/attempt context here to build one).""" + return self._corrupt_since_offset is not None + + @property + def corruption_offset(self) -> int | None: + return self._corrupt_since_offset + + def _recover(self) -> None: + # NOTE: the sequenced branch below must run even when the log is missing or empty -- an + # M2-style lost file (durable watermark, vanished log) is exactly the case M5 needs to + # reconcile against; an early return here would skip that reconciliation entirely and + # silently reset the allocator to 1. + records: list[tuple[int, bytes, Any]] = [] + if self._path.exists(): + data = self._path.read_bytes() + if data: + records, valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: + # N8: never truncate mid-file damage -- leave the bytes exactly as they are, + # trust only what came before, and let construction succeed anyway (B1 already + # forbids renumbering past it; the OTHER extreme -- raising here -- would + # discard every future emit, including the terminal event, forever). + self._report_corruption(corruption_offset) + elif valid_length < len(data): + with self._path.open("r+b") as stream: + stream.truncate(valid_length) + stream.flush() + os.fsync(stream.fileno()) # MIN-11: durable, not left as a crash window + if self._sequenced: + max_sequence = 0 + offsets: dict[int, int] = {} + for offset, _line, record in records: + if isinstance(record, dict): + sequence = record.get("sequence") + if isinstance(sequence, int): + offsets[sequence] = offset + if sequence > max_sequence: + max_sequence = sequence + self._offset_by_sequence = offsets + watermark = self.watermark() + if watermark > max_sequence: + logger.warning( + "%s: watermark (%s) is ahead of the highest sequence found in the spool (%s) " + "-- the log lost records the platform already processed; seeding " + "next_sequence from the watermark so newly allocated sequences don't collide " + "with ones the platform already closed", + self._name, watermark, max_sequence, + ) + self._next_sequence = max(max_sequence, watermark) + 1 + + def _truncate_to(self, size: int) -> None: + """B1: a true no-op on a failed append. `_next_sequence` is only advanced AFTER a + successful write, so the retry reuses the same sequence number -- this makes sure it reuses + clean ground too, instead of appending immediately after torn bytes with no `\\n` between + them (which would merge into one unparseable line the rest of this class can't recover + from).""" + try: + with self._path.open("r+b") as stream: + stream.truncate(size) + stream.flush() + os.fsync(stream.fileno()) + except OSError as exc: + # N12: the write may have failed before the file even existed (nothing to truncate, + # harmless) OR the rollback itself failed on an existing torn write -- in the latter + # case B1's "a failed append is a true no-op" no longer holds, so poison this spool + # rather than let a future append silently merge into the torn bytes. + self._poisoned = True + logger.error( + "%s: rollback of a failed append could not truncate the spool back to %d bytes " + "(%s) -- the file may now carry torn bytes; poisoning this spool so a caller sees " + "a typed error instead of a future append compounding the corruption", + self._name, size, exc, + ) + + def append(self, record: dict[str, Any]) -> SpooledRecord: + with self._lock: + self._require_writable() + if self._sequenced: + assigned = self._next_sequence + record = {**record, "sequence": assigned} + elif "sequence" in record: + raise OutboundSpoolError( + "outbound_spool_caller_supplied_sequence", + f"{self._name} spool does not assign sequence numbers; caller must not pass one", + ) + body = canonical_bytes(record) + if b"\n" in body: + # Framing invariant `_iter_complete_records`/`_recover` depend on: canonical JSON + # never contains a raw newline (json.dumps escapes control characters inside + # strings), so this would only fire on a value this module's own canonicalization + # contract disallows. + raise OutboundSpoolError( + "outbound_spool_record_unframable", f"{self._name}: record contains a raw newline" + ) + existed_before = self._path.exists() + size_before = self._path.stat().st_size if existed_before else 0 + try: + with self._path.open("ab") as stream: + stream.write(body) + stream.write(b"\n") + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + self._truncate_to(size_before) + raise + if not existed_before and not self._dir_synced: + # M2: the directory entry for a brand-new file isn't durable just because the + # file's own data is -- fsync it once (not per append; existing files' entries were + # already synced by whichever append first created them). + self._fsync_dir() + self._dir_synced = True + if self._sequenced: + self._offset_by_sequence[assigned] = size_before + self._next_sequence = assigned + 1 + return SpooledRecord(sequence=assigned, body=body) + return SpooledRecord(sequence=None, body=body) + + def records(self) -> list[SpooledRecord]: + """Reads the WHOLE log. The read itself happens OUTSIDE `self._lock` (M4): only the size + snapshot that bounds it is taken under the lock, so the flusher's (potentially large) read + never blocks `append`, the call loop's write path, for its duration. Safe because `append` + only ever grows the file -- a read bounded to a size captured a moment earlier can only be + stale, never torn. (`compact_through`/`drop` DO shrink the file and take the lock for their + entire duration; this module assumes the flusher serializes its own reads against its own + compactions rather than running them from two different threads.) + """ + with self._lock: + if not self._path.exists(): + return [] + size = self._path.stat().st_size + with self._path.open("rb") as stream: + data = stream.read(size) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: # N8: degrade to the readable prefix, never raise here + self._report_corruption(corruption_offset) + out: list[SpooledRecord] = [] + for _offset, line, decoded in parsed: + sequence = decoded.get("sequence") if self._sequenced and isinstance(decoded, dict) else None + out.append(SpooledRecord(sequence=sequence, body=line)) + return out + + def records_after(self, sequence: int) -> list[SpooledRecord]: + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + return [item for item in self.records() if (item.sequence or 0) > sequence] + + @property + def next_sequence(self) -> int: + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + assert self._next_sequence is not None + return self._next_sequence + + def watermark(self) -> int: + """The highest-processed sequence acknowledged so far ("the watermark is highest-processed + -- accepted AND rejected sequences both advance it"). Durable across a restart via a + fsync'd-temp-then-rename-then-fsync'd-directory write (M1) -- a corrupt or unreadable + watermark file DEGRADES TO 0 with a loud diagnostic rather than raising and wedging the + spool: re-sending already-acked events is safe (at-least-once delivery + platform-side + dedupe on `event_id`), while a permanently unreadable outbound channel is not. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + if not self._watermark_path.exists(): + return 0 + try: + raw = json.loads(self._watermark_path.read_text(encoding="utf-8")) + return int(raw["acked_through_sequence"]) + except (OSError, ValueError, KeyError, TypeError) as exc: + logger.warning( + "%s: watermark file is corrupt or unreadable (%s) -- degrading to 0. Re-sending " + "already-acked events is safe (at-least-once + dedupe on event_id); wedging the " + "spool permanently is not.", + self._name, exc, + ) + return 0 + + def advance_watermark(self, sequence: int) -> None: + """v1.3: `acked_through_sequence` is untrusted platform input. A value outside + `[current_watermark, next_sequence)` is rejected locally with a typed error and the + watermark is left exactly as it was -- "a malformed ack must not be able to discard + pending records" (M7). `sequence == current_watermark` is a legitimate no-op, not an error + (repeating the same ack, or a same-valued out-of-order response). + + P7: this is the one operation that DESTROYS delivery state (it durably advances what a + future `pending_since_watermark()` will ever return again) -- a forked child or a poisoned + instance advancing it would silently orphan every pending record below the new value, the + N1 outcome by a different route. `_require_writable()` guards it for that reason even + though nothing here writes to the JSONL log itself. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + with self._lock: + self._require_writable() + current = self.watermark() + if sequence < current or sequence >= self._next_sequence: + raise OutboundSpoolError( + "outbound_spool_watermark_out_of_range", + f"{self._name}: acked_through_sequence={sequence} outside the trusted range " + f"[{current}, {self._next_sequence}) -- untrusted platform input, watermark " + f"left unchanged", + ) + if sequence == current: + return + temporary = self.root / f"{self._name}.spool.watermark.tmp.{os.getpid()}.{uuid.uuid4().hex}" + with temporary.open("wb") as stream: + stream.write( + json.dumps({"acked_through_sequence": sequence}, separators=(",", ":")).encode( + "utf-8" + ) + ) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self._watermark_path) + self._fsync_dir() + + def pending_since_watermark(self) -> list[SpooledRecord]: + """Convenience for "the guest advances its spool cursor through the watermark": every + spooled record not yet acknowledged, in sequence order. Uses the offset `append` recorded + for the first pending sequence to SEEK directly there (M4) instead of re-reading and + re-parsing the whole log on every flush cycle; falls back to the generic full scan when no + cached offset exists yet (e.g. a fresh recovery whose watermark sits past every record this + process itself has written an offset for). + + N13: the drained steady state (nothing pending -- what a polling flusher sees most cycles) + is checked first and returns `[]` with NO file IO at all: `watermark + 1 == next_sequence` + means every allocated sequence has already been acknowledged, so there is nothing on disk + to seek to regardless of what `_offset_by_sequence` does or doesn't have cached. + """ + watermark = self.watermark() + with self._lock: + if watermark + 1 == self._next_sequence: + return [] + offset = self._offset_by_sequence.get(watermark + 1) + if offset is None: + return self.records_after(watermark) + with self._lock: + if not self._path.exists(): + return [] + size = self._path.stat().st_size + if offset >= size: + return [] + with self._path.open("rb") as stream: + stream.seek(offset) + data = stream.read(size - offset) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: # N8: absolute offset -- `data` starts at `offset` + self._report_corruption(offset + corruption_offset) + return [ + SpooledRecord(sequence=decoded.get("sequence") if isinstance(decoded, dict) else None, body=line) + for _offset, line, decoded in parsed + ] + + def _rewrite_retaining(self, keep: Callable[[Any], bool]) -> None: + """Shared by `compact_through` and `drop_many`: rewrites the log keeping only records + `keep` accepts, via the same write-fsync/atomic-replace/fsync-directory durability shape + `append`/`advance_watermark` use -- a crash mid-rewrite leaves either the old file intact + or the new one complete, never a torn hybrid. + + P1: if the log is already corrupt (N8), this REFUSES instead of rewriting. A rewrite always + replaces the file from what it read, and reading stops at the corruption offset -- so + rewriting on a corrupt spool would not merely skip the corrupt bytes, it would silently + destroy every intact record PAST them too, including ones this process itself appended + after recovery that have never been sent (the terminal event, in the worst case). That + directly defeats N8's "the readable prefix stays usable, delivery keeps working" posture + the moment the first drop/compact happens. Refusing costs only unbounded disk growth until + the attempt ends (`compact_through`'s whole job) or a rejected record staying spooled but + never re-emitted anyway, since it's at or below the watermark (`drop_many`'s whole job) -- + both strictly better than deleting undelivered records. + """ + with self._lock: + self._require_writable() + if not self._path.exists(): + return + size = self._path.stat().st_size + with self._path.open("rb") as stream: + data = stream.read(size) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: + self._report_corruption(corruption_offset) + logger.error( + "%s: refusing to compact/drop on a corrupt spool -- a rewrite replaces the file " + "from what it read, and reading stops at byte %d, so every record past that " + "offset (including not-yet-delivered ones) would be destroyed. The log is left " + "intact and grows unbounded until the attempt ends; that is the fail-safe half " + "of degrade-not-wedge.", + self._name, corruption_offset, + ) + return + temporary = self.root / f"{self._name}.spool.jsonl.tmp.{os.getpid()}.{uuid.uuid4().hex}" + offsets: dict[int, int] = {} + offset = 0 + with temporary.open("wb") as stream: + for _old_offset, line, decoded in parsed: + if not keep(decoded): + continue + if self._sequenced and isinstance(decoded, dict) and isinstance(decoded.get("sequence"), int): + offsets[decoded["sequence"]] = offset + stream.write(line) + stream.write(b"\n") + offset += len(line) + 1 + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self._path) + self._fsync_dir() + if self._sequenced: + self._offset_by_sequence = offsets + + def compact_through(self, sequence: int) -> None: + """M4: physically drops every durably-acked record (`sequence <= min(sequence, + watermark())`) from the on-disk log, bounding its growth for a long `running` stage. The + allocator's `next_sequence` is unaffected -- it is only ever derived from the log at + `_recover` time, and recovery's own `max(max_sequence, watermark)` rule (M5) already + tolerates a log whose historical records were compacted away, since none of them can be + the true maximum (compaction only ever removes sequences at or below the watermark, and + the watermark is always <= every pending, uncompacted sequence). + + Clamped to the current watermark regardless of what the caller passes -- compacting past + an event the platform hasn't actually processed yet would be irreversible data loss, and + this module's posture throughout is fail-safe over trusting the caller. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + effective = min(sequence, self.watermark()) + self._rewrite_retaining( + lambda decoded: not ( + isinstance(decoded, dict) + and isinstance(decoded.get("sequence"), int) + and decoded["sequence"] <= effective + ) + ) + + def drop_many(self, sequences: Collection[int]) -> None: + """The contract's rejected-event mechanism: "a rejected event is dropped from the spool ... + it is never re-emitted" (M5). PURE physical removal (N1) -- every sequence in `sequences` + is deleted from the on-disk log in ONE rewrite pass (N14: a batch of 100 rejections is one + `_rewrite_retaining` call, not 100), and the watermark is left untouched. + + N1: an earlier version had `drop` also advance the watermark to `sequence`, on the theory + that "a rejected event closes its sequence." That let an UNTRUSTED `rejected[].sequence` + from the platform silently orphan every pending record below it, bypassing + `advance_watermark`'s own M7 clamp entirely -- the clamp only guards `acked_through_sequence` + callers, and `drop` skipped straight past it. The batch-level + `advance_watermark(acked_through_sequence)` a caller performs separately is the ONE place + the watermark ever moves; it already covers every rejected sequence under a conformant + platform (rejections advance the watermark by contract), and under a non-conformant one + M7's clamp is then the single, correct chokepoint -- this method has no clamp of its own to + bypass. + + Writing the record's payload to the artifact spool as a `log` kind (the other half of the + contract's drop rule) is NOT this method's job -- that hand-off needs scenario/attempt + context and an `ArtifactsClient` this layer doesn't own; it is P9/P10 wiring, documented + here as the seam rather than guessed at. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + sequence_set = set(sequences) + if not sequence_set: + return + self._rewrite_retaining( + lambda decoded: not (isinstance(decoded, dict) and decoded.get("sequence") in sequence_set) + ) + + def drop(self, sequence: int) -> None: + """Single-sequence convenience wrapper over `drop_many` -- see its docstring for why this + no longer touches the watermark.""" + self.drop_many((sequence,)) + + +def _poison_after_fork() -> None: + """N25: `os.fork()` inherits both `OutboundSpool._registry` (with a live `_next_sequence`) and + every instance's flock fd (the SAME open file description, so the lock is merely shared, not + contended, across parent and child) -- without this, parent and child would allocate identical + sequence numbers with no complaint. Marks every currently-registered instance so its next + mutating call raises instead. Not reachable via `subprocess` (fork+exec resets memory); this + guards a bare `os.fork()` specifically.""" + with OutboundSpool._registry_lock: + for instance in OutboundSpool._registry.values(): + instance._forked = True + + +if hasattr(os, "register_at_fork"): # POSIX-only, like fcntl (N30) + # P7: `before=`/`after_in_parent=` pair the registry lock around the fork itself -- without + # this, a fork occurring while some OTHER thread holds `_registry_lock` hands the child a + # locked RLock owned by a thread that no longer exists there, and `_poison_after_fork`'s own + # `with OutboundSpool._registry_lock:` deadlocks at the fork point instead of poisoning + # anything. Acquiring on `before` guarantees the FORKING thread itself owns the lock at fork + # time, so the child's single surviving thread already owns it too -- `_poison_after_fork`'s + # acquire becomes a safe reentrant no-op there, and `after_in_parent` restores normal locking + # in the parent. + os.register_at_fork( + before=OutboundSpool._registry_lock.acquire, + after_in_parent=OutboundSpool._registry_lock.release, + after_in_child=_poison_after_fork, + ) + + +# ================================================================================================= +# Transport -- the HTTP boundary every channel client speaks through, and its production impl. +# ================================================================================================= + + +@dataclass(frozen=True) +class TransportResponse: + status_code: int + body: dict[str, Any] | None + headers: dict[str, str] + + +class TransportError(RuntimeError): + """Raised by a `Transport.request` implementation when no HTTP response was ever received + (connection refused, DNS failure, timeout, ...). `classify_response` treats this identically + to an unreachable 5xx -- the guest cannot distinguish "server errored" from "server unreachable" + and the contract's retry policy doesn't ask it to.""" + + +class Transport(Protocol): + """The seam every channel client is built against, so the fake-platform tests exercise the + exact same code path production traffic does -- only what sits behind this protocol differs. + """ + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Iterator[bytes] | None = None, + timeout: float = 30.0, + ) -> TransportResponse: ... + + +class RequestsTransport: + """Production `Transport`: a thin `requests.Session` wrapper. Every network-layer failure + (`requests.RequestException`, which covers connection errors, timeouts, and retries `requests` + itself doesn't handle) is normalized to `TransportError` so `classify_response` never needs to + know which HTTP library is underneath.""" + + def __init__(self, *, session: requests.Session | None = None) -> None: + self._session = session or requests.Session() + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Iterator[bytes] | None = None, + timeout: float = 30.0, + ) -> TransportResponse: + try: + response = self._session.request( + method, url, headers=headers, json=json_body, data=data, timeout=timeout + ) + except requests.RequestException as exc: + raise TransportError(str(exc)) from exc + try: + body = response.json() if response.content else None + except ValueError: + body = None + return TransportResponse( + status_code=response.status_code, body=body, headers=dict(response.headers) + ) + + +def _iter_chunks(data: bytes, chunk_size: int) -> Iterator[bytes]: + """§3a: "Uploads >64 MB use chunked transfer." A fresh generator is built per send attempt + (never reused across a retry) -- a generator is single-use, and reusing an exhausted one would + silently upload an empty body on the second attempt.""" + for start in range(0, len(data), chunk_size): + yield data[start : start + chunk_size] + + +def _parse_retry_after(headers: Mapping[str, str] | None) -> float | None: + """"429 -> honor `Retry-After`." Only the delta-seconds form is parsed (the integer count of + seconds to wait) -- the contract never mentions the alternative HTTP-date form and every + platform emitter in this ecosystem is expected to send the simple form; an unparseable value is + treated as absent so the caller falls back to the computed backoff rather than crashing. + + N5: a negative value is ALSO treated as absent -- `time.sleep(-5)` raises `ValueError`, and a + server sending a negative `Retry-After` is malformed input this module owes no obedience to. + The upper clamp (`[0, retry_policy.max_backoff_seconds]`) needs the policy, which isn't + available here -- `_perform_with_retry` applies that half. + + P6: header-name lookup is case-INsensitive (RFC 9110 §5.1 -- field names are case-insensitive). + `RequestsTransport` builds `dict(response.headers)` from `requests`' own `CaseInsensitiveDict`, + which drops the case-insensitivity and preserves whatever casing the server actually sent -- a + plain `.get("Retry-After")` would miss `RETRY-AFTER`/`Retry-after` and silently fall back to + computed backoff instead of honoring the server's wait. + """ + if not headers: + return None + value = next((v for k, v in headers.items() if k.lower() == "retry-after"), None) + if value is None: + return None + try: + parsed = float(value) + except ValueError: + return None + return parsed if parsed >= 0 else None + + +# ================================================================================================= +# Error map -- the contract's closed status-code vocabulary ("Error responses" / "Failure semantics +# summary"), and the shared retry engine every channel client drives it through. +# ================================================================================================= + + +class ChannelOutcome(str, Enum): + """Every way one outbound HTTP attempt can resolve, per the contract's failure table. Not a + contract vocabulary itself (the wire only ever carries a status code + `{error, message, + retryable}`) -- this is this module's own closed classification of that table, the thing + `classify_response` computes and every client branches on.""" + + DELIVERED = "delivered" + RETRYABLE = "retryable" + FENCED = "fenced" + PERMANENT_ITEM = "permanent_item" + CHANNEL_FAILED = "channel_failed" + BUDGET_EXCEEDED = "budget_exceeded" + + +@dataclass(frozen=True) +class ChannelError: + outcome: ChannelOutcome + domain: FailureDomain | None + code: str + message: str + retry_after_seconds: float | None = None + # N7: the raw status this was classified from (`None` for a `TransportError`/no-response + # outcome) -- carried so a caller can recognize a specific status (413, for the events-batch + # halving retry) without `ChannelOutcome`/`code` alone being expressive enough for that. + status_code: int | None = None + + +class HostedFencedError(RuntimeError): + """401 (expired) / 403 (fence, scope, mismatch): "stop emitting, exit code 3 ... never an infra + retry." Raised by the shared retry engine and never retried -- the entrypoint (outside this + module) is the one that translates this into the process exit code.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class HostedChannelFailedError(RuntimeError): + """404, retried 3x per the contract, still 404: "finalize `platform_sync`." Raised by the + shared retry engine once `classify_response` reaches the third 404 attempt.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class HostedAttemptSupersededError(RuntimeError): + """N22: `409 attempt_superseded` folded into the `ChannelState` latch below -- "a fenced + attempt's in-flight requests cannot land after registration of its successor" is a fence in + substance, even though the ONE request that received it is still correctly classified + `PERMANENT_ITEM` (contract-correct: 409 is item-level, not fence-level). Only raised by + `ChannelState.check()` on a LATER call, once a prior call has already seen this code -- the + call that actually observed the 409 still returns its normal item-level result.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class ChannelState: + """N10: shared "stop emitting" latch across the three channel clients for one attempt -- a + fence (401/403) or an exhausted channel (404x3) on ANY channel must stop ALL of them, since + the token/fence is per-ATTEMPT, not per-channel ("stop emitting ... never an infra retry"). + Also carries the N22 attempt-supersession latch (409 `attempt_superseded`). + + Construct ONE `ChannelState` per attempt and pass it to `EventsClient`/`ResultsClient`/ + `ArtifactsClient` alike (each defaults to a private one if not given, which only latches + itself -- correct for a single-channel caller, but callers driving more than one channel for + the same attempt MUST share one instance to get the cross-channel guarantee this class exists + for). Once latched, `check()` raises the SAME error on every subsequent call, from any client + sharing this state, without ever touching the transport. + """ + + def __init__(self) -> None: + self._error: HostedFencedError | HostedChannelFailedError | HostedAttemptSupersededError | None = None + self._lock = RLock() + + def check(self) -> None: + with self._lock: + error = self._error + if error is not None: + raise error + + def latch( + self, exc: "HostedFencedError | HostedChannelFailedError | HostedAttemptSupersededError" + ) -> None: + with self._lock: + if self._error is None: + self._error = exc + + +def classify_response( + status_code: int | None, + body: dict[str, Any] | None, + *, + attempt: int, + retry_after_seconds: float | None = None, +) -> ChannelError | None: + """The single call site every channel client classifies a transport outcome through. Returns + `None` for a delivered (2xx) response, a `ChannelError` for everything else. `status_code=None` + means `TransportError` (no response was ever received) -- classified exactly like an + unreachable 5xx. For the 404 branch specifically, `attempt` must be the count of 404 RESPONSES + seen so far in this call (not the overall attempt number) -- `_perform_with_retry` tracks that + counter separately (N6) so an interleaved 5xx never shortens the 404 budget; every other branch + ignores `attempt` entirely. + + N28: the error body's `retryable` field is deliberately never read here -- classification is + status-keyed throughout this module (every branch below is keyed on `status_code` alone), which + is defensible per the contract's own closed, status-code-driven failure table; if the platform + ever marks an unexpected status `retryable` the guest disagrees silently, a known, accepted gap. + + Every branch below is transcribed directly from the contract's "Error responses" paragraph and + "Failure semantics summary" table -- see `outbound-channels.md` v1.3 for the prose this mirrors. + """ + if status_code is not None and 200 <= status_code < 300: + return None + + error_code = (body or {}).get("error") if body else None + message = (body or {}).get("message", "") if body else "" + + if status_code is None: + return ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + error_code or "network_error", + message or "transport failure: no response received", + status_code=None, + ) + if status_code in (401, 403): + return ChannelError(ChannelOutcome.FENCED, None, error_code or "fenced", message, status_code=status_code) + if status_code == 404: + # N29: PLATFORM_SYNC on every 404 attempt, not just the third -- a 404 is never a + # connectivity fault under §4.6; only the third attempt's outcome is ever surfaced to a + # caller, but the domain should not silently disagree across attempts 1-2 vs 3. + domain = FailureDomain.PLATFORM_SYNC + if attempt < 3: + return ChannelError(ChannelOutcome.RETRYABLE, domain, error_code or "not_found", message, status_code=404) + return ChannelError( + ChannelOutcome.CHANNEL_FAILED, domain, error_code or "not_found", message, status_code=404 + ) + if status_code == 413: + # N7: 413 is Channel 3's artifact-budget code specifically -- only classify it + # BUDGET_EXCEEDED when the body actually says so; a 413 on any other channel (e.g. an + # events batch that simply exceeded the platform's ingress size limit) falls through to + # the unlisted-4xx catch-all below instead of being mislabeled as a budget condition that + # channel doesn't have. + if error_code == "artifact_budget_exceeded": + return ChannelError( + ChannelOutcome.BUDGET_EXCEEDED, None, error_code, message, status_code=413 + ) + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or "http_413", message, status_code=413 + ) + if status_code == 429: + return ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + error_code or "rate_limited", + message, + retry_after_seconds=retry_after_seconds, + status_code=429, + ) + if status_code in (400, 409, 422): + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + if 500 <= status_code < 600: + return ChannelError( + ChannelOutcome.RETRYABLE, FailureDomain.CONNECTIVITY, error_code or "server_error", message, + status_code=status_code, + ) + if 400 <= status_code < 500: + # "Catch-all: any unlisted 4xx is permanent for that item." + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + # N27: no other status family is contractual -- notably a 3xx, which should never occur (every + # endpoint ends in "/" precisely so Django's POST-redirect problem never arises). Treat as + # permanent rather than retrying an endpoint misconfiguration `max_attempts` times before + # giving up anyway. + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + + +def compute_backoff_seconds( + attempt: int, + *, + initial_backoff_seconds: float, + max_backoff_seconds: float, + rng: Callable[[], float] = random.random, +) -> float: + """"retry with backoff (base `retry.initial_backoff_seconds`, cap `retry.max_backoff_seconds`, + full jitter)" -- `uniform(0, min(cap, base * 2**(attempt-1)))`. `attempt` is the 1-based attempt + that just failed. `rng` is injectable so callers (and tests) can get a deterministic value. + """ + ceiling = min(max_backoff_seconds, initial_backoff_seconds * (2 ** max(0, attempt - 1))) + return rng() * ceiling + + +@dataclass(frozen=True) +class RetryPolicy: + """Backoff parameters, shared by all three channel clients. Field names mirror + `job.HarnessRetryPolicy` deliberately (the contract states these come from the job's own + `retry.initial_backoff_seconds`/`retry.max_backoff_seconds`), but this module does not import + that class -- a client only needs two floats, and importing the full job-retry model (with its + `retryable_domains` field, which governs WHOLE-JOB attempt retries, a distinct concept from a + single outbound delivery's backoff) would be a coupling this module doesn't need. + + `max_attempts` bounds one `_perform_with_retry` call's own retry loop for the classes the + contract leaves unbounded (network/5xx/429 -- "spool + backoff", no stated attempt ceiling). + STUCK DECISION (fail-safe/reversible, contract silent): capped at a generous default (8) rather + than looped forever, because durability already lives in the spool/idempotent-wire-design, not + in one blocking call -- a caller that wants to keep trying simply invokes the client method + again later (`EventsClient.flush()` is designed to be called repeatedly for exactly this + reason). Must stay >= 3 for the 404 rule to ever reach its own `CHANNEL_FAILED` transition + within a single call; the default comfortably clears that. + """ + + initial_backoff_seconds: float = 1.0 + max_backoff_seconds: float = 15.0 + max_attempts: int = 8 + + +def _perform_with_retry( + perform: Callable[[int], TransportResponse], + *, + retry_policy: RetryPolicy, + sleep: Callable[[float], None], + rng: Callable[[], float] = random.random, + deadline: float | None = None, + now: Callable[[], float] = time.monotonic, +) -> tuple[TransportResponse | None, ChannelError | None]: + """The one retry/backoff engine all three channel clients drive their single HTTP call through. + `perform(attempt)` makes ONE attempt (1-based); this loops it, classifies each outcome via + `classify_response`, and: + + - returns `(response, None)` once delivered; + - raises `HostedFencedError` immediately on 401/403 (never retried, by contract); + - raises `HostedChannelFailedError` once a 404 reaches its third attempt; + - returns `(response_or_None, error)` for every other terminal outcome (`PERMANENT_ITEM`, + `BUDGET_EXCEEDED`) without retrying -- "deterministic rejections are never retried"; + - otherwise (`RETRYABLE`) sleeps -- honoring a server `Retry-After` over the computed backoff + when present -- and tries again, up to `retry_policy.max_attempts`. + + N5/P5: `deadline` (a `time.monotonic()` value, typically the adapter's flush-window end) bounds + ATTEMPT SCHEDULING AND SLEEPS ONLY -- no new attempt starts once `now() >= deadline`, and every + sleep (computed backoff OR a server `Retry-After`) is clamped to whatever budget remains. It + does NOT bound an attempt's own in-flight request: `perform`'s transport call carries its own, + separate `timeout` (the caller's second knob -- `Transport.request`'s `timeout` parameter, + unrelated to `deadline`), and nothing here clamps that value against the remaining deadline + budget. A caller sizing `deadline` against a hard wall-clock guarantee for the whole call is + sizing against a guarantee this function does not provide; only "no new attempt starts, and no + sleep runs, once the window is gone" is guaranteed. (P5: widening `perform` to accept a clamped + per-attempt timeout was considered and is the more complete fix, but at least one caller outside + this module -- `hosted_entrypoint.py`'s `ScenariosClient._post`, which calls this function + directly with its own single-argument `perform` closure -- is out of this fix's scope, so + changing the call signature here would silently break that caller instead of fixing it. This + docstring correction is the floor the round-3 review named for exactly that situation.) + + N6: 404 retries are counted SEPARATELY from the overall attempt number (`not_found_attempts`), + so an interleaved 5xx (e.g. `503, 503, 404, 404, 404`) does not shorten the 404 budget -- only + three OBSERVED 404s reach `classify_response`'s `CHANNEL_FAILED` transition, regardless of what + else happened in between. + """ + attempt = 0 + not_found_attempts = 0 + while True: + attempt += 1 + if deadline is not None and now() >= deadline: + return None, ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + "deadline_exceeded", + "the flush-window deadline elapsed before this attempt could be made", + ) + try: + response = perform(attempt) + except TransportError: + error = classify_response(None, None, attempt=attempt) + response = None + else: + status = response.status_code + if status == 404: + not_found_attempts += 1 + error = classify_response( + status, + response.body, + attempt=not_found_attempts if status == 404 else attempt, + retry_after_seconds=_parse_retry_after(response.headers), + ) + if error is None: + return response, None + if error.outcome is ChannelOutcome.FENCED: + raise HostedFencedError(error) + if error.outcome is ChannelOutcome.CHANNEL_FAILED: + raise HostedChannelFailedError(error) + if error.outcome is not ChannelOutcome.RETRYABLE: + return response, error + if attempt >= retry_policy.max_attempts: + return response, error + delay = error.retry_after_seconds + if delay is not None: + delay = min(delay, retry_policy.max_backoff_seconds) # N5: clamp Retry-After + else: + delay = compute_backoff_seconds( + attempt, + initial_backoff_seconds=retry_policy.initial_backoff_seconds, + max_backoff_seconds=retry_policy.max_backoff_seconds, + rng=rng, + ) + if deadline is not None: + remaining = deadline - now() + if remaining <= 0: + return response, error + delay = min(delay, remaining) + sleep(delay) + + +# ================================================================================================= +# Channel 1 -- Events client. Batches spooled events, advances the watermark only on confirmed +# delivery. +# ================================================================================================= + + +@dataclass(frozen=True) +class EventsFlushResult: + delivered_count: int + acked_through_sequence: int | None + rejected: list[dict[str, Any]] + error: ChannelError | None + # v1.3 (M7): set when the platform's `acked_through_sequence` fell outside the spool's trusted + # range and was ignored rather than trusted -- `error` stays `None` because the HTTP delivery + # itself succeeded; only the ack body was untrustworthy. + ack_out_of_range: bool = False + # N2/N3: set when a 2xx response's `acked_through_sequence` was missing, `null`, or not an + # int -- a protocol violation distinct from "present but out of range" above. `error` stays + # `None` for the same reason: the HTTP delivery itself succeeded. + ack_missing: bool = False + # P9: the SpooledRecord bodies dropped this call (a subset of `batch`, keyed by `rejected`), + # captured BEFORE `drop_many` removes them from disk. The contract requires a rejected event's + # payload be "written to the artifact spool (`log` kind)" -- without this, a caller has no way + # to recover that payload at all once `flush()` returns, since the cap applied inside `flush()` + # makes it impossible to reliably re-derive which spooled records were even in this batch. + dropped_records: list[SpooledRecord] = field(default_factory=list) + + +_EVENTS_BATCH_PREFIX = b'{"schema_version":"' + EVENT_SCHEMA_VERSION.encode("utf-8") + b'","events":[' +_EVENTS_BATCH_SUFFIX = b"]}" + + +def _encode_events_batch(records: list[SpooledRecord]) -> bytes: + """N19: the contract says "serialize once, spool the bytes, re-send verbatim; never + re-serialize on retry" -- for the events BATCH ENVELOPE itself, not just each event inside it. + Handing a decoded dict to `json_body=` (the previous shape) let `requests` re-serialize the + envelope with its own settings on every send; this instead concatenates the exact bytes + `OutboundSpool.append` already wrote for each event, closing the deviation rather than merely + documenting it. Safe because `EVENT_SCHEMA_VERSION` is a fixed ASCII constant with no bytes + needing escape.""" + return _EVENTS_BATCH_PREFIX + b",".join(record.body for record in records) + _EVENTS_BATCH_SUFFIX + + +class EventsClient: + """Delivers `OutboundSpool`-backed Channel 1 events to `endpoints.events`. One `flush()` call + sends one batch (<= `EVENTS_MAX_BATCH` events, <= `EVENTS_MAX_BATCH_BYTES`) of everything + spooled since the last confirmed watermark, in spool order + (`OutboundSpool.pending_since_watermark`, which reads the log in append/sequence order -- the + platform's own ordering rule, "`(attempt_number, sequence)`", so a single-attempt process + satisfies it for free). Re-sends spooled bytes verbatim (N19, `_encode_events_batch`) -- never + recomputing an event's own `digest`, so "serialize once ... never re-serialize on retry" holds + for the one thing that must never drift (the per-event digest, embedded as data). + + `flush()` is meant to be called repeatedly (by whatever background loop owns the call cadence, + a scheduler concern outside this module) -- each call is a complete, self-contained delivery + attempt (with its own internal retry/backoff via `_perform_with_retry`) that advances the + watermark exactly as far as the platform confirmed and leaves everything else spooled for the + next call. + + The ack body is UNTRUSTED PLATFORM INPUT end to end (v1.3): `acked_through_sequence` goes + through the spool's own M7 clamp (`advance_watermark`); `rejected[]` is filtered to sequences + actually present in the batch just sent BEFORE anything is done with it (N1) -- a value the + guest never sent cannot cause a drop, and dropping never itself advances the watermark (see + `OutboundSpool.drop_many`) -- so the batch-level `advance_watermark(acked_through_sequence)` is + the single chokepoint either way. + """ + + def __init__( + self, + capabilities: HostedCapabilities, + spool: OutboundSpool, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + batch_size: int = EVENTS_MAX_BATCH, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._spool = spool + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._batch_size = max(1, min(batch_size, EVENTS_MAX_BATCH)) + self._channel_state = channel_state or ChannelState() + + def _cap_batch(self, records: list[SpooledRecord]) -> list[SpooledRecord]: + """Proactive half of N7: cap by event count (`_batch_size`) AND cumulative canonical bytes + (`EVENTS_MAX_BATCH_BYTES`) before ever building a request -- reduces how often the reactive + 413-halving in `flush()` below is ever needed. Always includes at least one record (its own + oversized payload is HostedEventDraft's problem, at spool-append time, not this cap's).""" + capped = records[: self._batch_size] + limited: list[SpooledRecord] = [] + total_bytes = 0 + for record in capped: + if limited and total_bytes + len(record.body) > EVENTS_MAX_BATCH_BYTES: + break + limited.append(record) + total_bytes += len(record.body) + return limited + + def flush(self, *, deadline: float | None = None) -> EventsFlushResult: + self._channel_state.check() + batch = self._cap_batch(self._spool.pending_since_watermark()) + if not batch: + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ) + + response: TransportResponse | None = None + error: ChannelError | None = None + while True: + body_bytes = _encode_events_batch(batch) + headers = {**self._capabilities.auth_headers(), "Content-Type": "application/json"} + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", self._capabilities.endpoints.events, headers=headers, data=body_bytes + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + # N7: a 413 on the events channel -- reactively halve the batch and try again rather + # than returning a permanent, non-progressing error for the whole thing. Stops once a + # single event alone still 413s (defensive; should not happen under EVENT_PAYLOAD_MAX_BYTES). + if error is not None and error.status_code == 413 and len(batch) > 1: + logger.warning( + "events flush: batch of %d events (%d bytes) was rejected with 413 -- halving " + "and retrying", len(batch), len(body_bytes), + ) + batch = batch[: len(batch) // 2] + continue + break + + if error is not None or response is None: + return EventsFlushResult( + delivered_count=0, acked_through_sequence=None, rejected=[], error=error + ) + + # N2/N3: the ack body is untrusted platform input, defensively parsed -- a wrong type or a + # missing key must never raise out of flush() (that would silence the flusher loop, B1's + # outcome by a different route) and must never be treated as "0" (that would silently + # re-send the same batch forever, N3). + body = response.body if isinstance(response.body, dict) else {} + raw_acked = body.get("acked_through_sequence") + acked_through: int | None + if isinstance(raw_acked, int) and not isinstance(raw_acked, bool): + acked_through = raw_acked + else: + acked_through = None + logger.warning( + "events flush: 2xx response has a missing/invalid acked_through_sequence (got %r) " + "-- treating as a protocol violation, not advancing the watermark", raw_acked, + ) + + raw_rejected = body.get("rejected") + if isinstance(raw_rejected, list): + rejected_entries = [entry for entry in raw_rejected if isinstance(entry, dict)] + if len(rejected_entries) != len(raw_rejected): + logger.warning("events flush: rejected[] contained non-object entries -- ignoring them") + else: + rejected_entries = [] + if raw_rejected is not None: + logger.warning( + "events flush: rejected is %r, not a list -- treating as empty", type(raw_rejected).__name__ + ) + + if acked_through is None: + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ack_missing=True, + ) + + # N1: filter rejected[] to sequences the guest ACTUALLY sent in this batch, before doing + # anything with them -- a sequence the platform names that was never in `batch` is + # untrusted input this module owes no obedience to (it cannot be dropped, since it was + # never spooled under that number in the first place, and trusting it would let a + # malformed ack orphan pending records by a route the M7 clamp doesn't guard). + batch_sequences = {record.sequence for record in batch if record.sequence is not None} + valid_rejected: list[dict[str, Any]] = [] + for entry in rejected_entries: + sequence = entry.get("sequence") + if isinstance(sequence, int) and not isinstance(sequence, bool) and sequence in batch_sequences: + valid_rejected.append(entry) + else: + logger.warning( + "events flush: rejected entry names sequence=%r, which was not sent in this " + "batch (sent=%s) -- ignoring as untrusted platform input", + sequence, sorted(batch_sequences), + ) + + # P9: capture the dropped records' own bodies BEFORE drop_many physically removes them -- + # once removed, this is the only place a caller can still recover the payload the contract + # requires be "written to the artifact spool (`log` kind)" for a rejected event. + rejected_sequences = {entry["sequence"] for entry in valid_rejected} + dropped_records = [record for record in batch if record.sequence in rejected_sequences] + + # N1/N14: pure physical removal, batched into one rewrite -- drop_many never touches the + # watermark; the batch-level advance_watermark(acked_through) below is the ONLY chokepoint. + self._spool.drop_many(rejected_sequences) + + try: + # "the watermark is highest-processed -- accepted AND rejected sequences both advance + # it." v1.3: `acked_through_sequence` is untrusted input -- the spool itself enforces + # the clamp (M7). + self._spool.advance_watermark(acked_through) + except OutboundSpoolError: + logger.warning( + "events flush: platform returned an untrusted acked_through_sequence=%s outside " + "the guest's trusted range -- ignoring the ack, watermark unchanged at %s", + acked_through, self._spool.watermark(), + ) + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ack_out_of_range=True, + dropped_records=dropped_records, + ) + + delivered_count = sum( + 1 + for record in batch + if record.sequence is not None + and record.sequence <= acked_through + and record.sequence not in rejected_sequences + ) + return EventsFlushResult( + delivered_count=delivered_count, + acked_through_sequence=acked_through, + rejected=valid_rejected, + error=None, + dropped_records=dropped_records, + ) + + +# ================================================================================================= +# Channel 2 -- Result receipts. Typed `ResultReceiptDraft` + delivery. +# ================================================================================================= + + +class ScenarioStatus(str, Enum): + PASSED = "passed" + FAILED = "failed" + ERRORED = "errored" + SKIPPED = "skipped" + + +class SubGoalResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + held: bool | None + reason: str | None + judged: bool + + +class MetricEvaluation(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal["metric"] + score: float = Field(ge=0.0, le=1.0) + reason: str + + +class CheckpointEvaluation(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal["checkpoint"] + passed: bool + reason: str + + +EvaluationResult = MetricEvaluation | CheckpointEvaluation + + +class CallSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + started_at: str + ended_at: str + duration_ms: int = Field(ge=0) + turns: int = Field(ge=0) + transcript_artifact: str | None + recording_artifacts: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _validate(self) -> "CallSummary": + for label, value in (("started_at", self.started_at), ("ended_at", self.ended_at)): + if not is_valid_rfc3339_millis(value): + raise ValueError(f"call_timestamp_invalid: {label}={value!r}") + if self.transcript_artifact is not None and not is_valid_digest(self.transcript_artifact): + raise ValueError(f"call_transcript_artifact_invalid: {self.transcript_artifact!r}") + for artifact in self.recording_artifacts: + if not is_valid_digest(artifact): + raise ValueError(f"call_recording_artifact_invalid: {artifact!r}") + return self + + +def _unset_default_fields(model: BaseModel, prefix: str = "") -> list[str]: + """N23: names the fields a caller did NOT explicitly set (filled from a pydantic default) -- + the actual shape of a digest-mismatch bug like the review's example: a raw `call` dict that + omits `recording_artifacts` computes an external digest over an object without that key, while + the model's own re-derivation fills in `recording_artifacts: []`. This is not a general diff + against the caller's original raw dict (a model validator has no access to that, only to what + pydantic recorded via `model_fields_set`) -- it is the honest, dotted-path subset available from + inside the model: which fields with defaults were left unset, one level into nested models + (covers `call.recording_artifacts`, not just top-level `world_index`/`schema_version`).""" + names: list[str] = [] + for name in type(model).model_fields: + if name == "digest": + continue + path = f"{prefix}{name}" + if name not in model.model_fields_set: + names.append(path) + value = getattr(model, name) + if isinstance(value, BaseModel): + names.extend(_unset_default_fields(value, f"{path}.")) + return names + + +class ResultReceiptDraft(BaseModel): + """Channel 2's wire shape. Mirrors `HostedEventDraft`'s pattern: the caller supplies `digest` + (via `build_result_receipt`, computed with `whole_object_digest`), the model re-derives and + rejects a mismatch, plus the two exact-shape rules the contract states as literal requirements + rather than general validation ("`skipped` receipt body (exact)" and "`errored` receipt body"). + """ + + model_config = ConfigDict(extra="forbid") + + schema_version: str = RESULT_SCHEMA_VERSION + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + scenario_key: str = Field(min_length=1) + scenario_id: str = Field(min_length=1) + scenario_attempt: Literal[1, 2] + world_index: int | None = Field(default=None, ge=0) + status: ScenarioStatus + sub_goals: list[SubGoalResult] + evaluations: list[EvaluationResult] + call: CallSummary | None + failure: TerminalFailure | None + digest: str + + @model_validator(mode="after") + def _validate(self) -> "ResultReceiptDraft": + if self.schema_version != RESULT_SCHEMA_VERSION: + raise ValueError(f"result_schema_unsupported: {self.schema_version}") + if not is_valid_digest(self.digest): + raise ValueError(f"receipt_digest_invalid: {self.digest!r}") + expected = whole_object_digest(self.model_dump(mode="json", exclude={"digest"})) + if self.digest != expected: + unset = _unset_default_fields(self) + hint = f" -- fields not explicitly set, filled from defaults: {', '.join(unset)}" if unset else "" + raise ValueError(f"receipt_digest_mismatch{hint}") + + if self.status is ScenarioStatus.SKIPPED: + if ( + self.scenario_attempt != 1 + or self.world_index is not None + or self.sub_goals + or self.evaluations + or self.call is not None + or self.failure is not None + ): + raise ValueError("skipped_receipt_shape_invalid") + elif self.status is ScenarioStatus.ERRORED and self.failure is None: + raise ValueError("errored_receipt_requires_failure") + return self + + +def build_result_receipt( + *, + job_id: str, + attempt_id: str, + attempt_number: int, + scenario_key: str, + scenario_id: str, + scenario_attempt: Literal[1, 2], + world_index: int | None, + status: ScenarioStatus, + sub_goals: list[dict[str, Any]], + evaluations: list[dict[str, Any]], + call: dict[str, Any] | None, + failure: dict[str, Any] | None, + extra_secret_values: tuple[str, ...] = (), +) -> dict[str, Any]: + """Validate one receipt's shape and compute its digest, returning a plain wire-ready dict. + Mirrors `build_event_record`'s contract exactly: callers must pass already wire-typed values + (e.g. a metric `score` as `float`, never `int`) -- the digest is computed on the RAW input + before model validation/coercion, so a type looseness here fails loudly as a digest mismatch + rather than silently spooling a digest that doesn't match what gets sent. + + N9: `redact_outbound_text` runs on `sub_goals[].reason`, `evaluations[].reason`, and + `failure.{code,message}` (P8) BEFORE the digest is computed -- same ordering rationale as + `build_event_record`: the embedded digest must match the redacted bytes actually sent. + """ + + def _redact(text: Any) -> Any: + return redact_outbound_text(text, extra_secret_values) if isinstance(text, str) else text + + sub_goals = [ + {**goal, "reason": _redact(goal.get("reason"))} if isinstance(goal, dict) else goal + for goal in sub_goals + ] + evaluations = [ + {**item, "reason": _redact(item.get("reason"))} if isinstance(item, dict) else item + for item in evaluations + ] + if isinstance(failure, dict): + updates = { + key: _redact(failure[key]) + for key in ("code", "message") + if isinstance(failure.get(key), str) + } + if updates: + failure = {**failure, **updates} + + core: dict[str, Any] = { + "schema_version": RESULT_SCHEMA_VERSION, + "job_id": job_id, + "attempt_id": attempt_id, + "attempt_number": attempt_number, + "scenario_key": scenario_key, + "scenario_id": scenario_id, + "scenario_attempt": scenario_attempt, + "world_index": world_index, + "status": status.value if isinstance(status, ScenarioStatus) else status, + "sub_goals": sub_goals, + "evaluations": evaluations, + "call": call, + "failure": failure, + } + digest = whole_object_digest(core) + draft = ResultReceiptDraft.model_validate({**core, "digest": digest}) + return draft.model_dump(mode="json") + + +def build_skipped_receipt( + *, job_id: str, attempt_id: str, attempt_number: int, scenario_key: str, scenario_id: str +) -> dict[str, Any]: + """The "exact" synthesized body for a scenario that never ran ("The guest synthesizes these + during the flush window; the finalizer backfills any still missing").""" + return build_result_receipt( + job_id=job_id, + attempt_id=attempt_id, + attempt_number=attempt_number, + scenario_key=scenario_key, + scenario_id=scenario_id, + scenario_attempt=1, + world_index=None, + status=ScenarioStatus.SKIPPED, + sub_goals=[], + evaluations=[], + call=None, + failure=None, + ) + + +@dataclass(frozen=True) +class ReceiptPushResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +class ResultsClient: + """Delivers one Channel 2 receipt per call to `endpoints.results`. No spool/watermark of its + own -- unlike events, receipts carry no `sequence`; their idempotency key is `(job_id, + scenario_key)` (contract), so redelivery safety comes from the wire protocol itself (`200` + duplicate on a matching digest) rather than from a local ack cursor. A caller that wants + durable at-least-once delivery across a process crash owns that queuing (e.g. an + `OutboundSpool(sequenced=False)`, exposed by this module for exactly this) and simply calls + `push()` again for anything not yet confirmed -- safe because the platform's own idempotency + check is what makes a redelivery a no-op, not any state this client keeps. + """ + + def __init__( + self, + capabilities: HostedCapabilities, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._channel_state = channel_state or ChannelState() + + def push(self, receipt: dict[str, Any], *, deadline: float | None = None) -> ReceiptPushResult: + self._channel_state.check() + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", + self._capabilities.endpoints.results, + headers=self._capabilities.auth_headers(), + json_body=receipt, + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None and error.code == "attempt_superseded": # N22 + self._channel_state.latch(HostedAttemptSupersededError(error)) + if error is not None or response is None: + return ReceiptPushResult(delivered=False, already_existed=False, error=error) + # N20: `200` is read as "already exists / duplicate" per the contract's idempotency rule; + # the contract never states the success code for a genuinely NEW receipt (this module's own + # `FakePlatform` uses `201`, unconfirmed against the real platform -- see the review report). + return ReceiptPushResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + +# ================================================================================================= +# Channel 3 -- Artifacts. Content-addressed upload + `ArtifactManifestDraft` + delivery. +# ================================================================================================= + + +class ArtifactKind(str, Enum): + RECORDING_COMBINED = "recording_combined" + RECORDING_STEREO = "recording_stereo" + RECORDING_CUSTOMER = "recording_customer" + RECORDING_ASSISTANT = "recording_assistant" + TRANSCRIPT = "transcript" + TOOL_TRACE = "tool_trace" + RESULT = "result" + BUILD = "build" + TRACE = "trace" + LOG = "log" + OTHER = "other" + + +_RESERVED_ARTIFACT_KINDS = frozenset( + {ArtifactKind.BUILD, ArtifactKind.TRANSCRIPT, ArtifactKind.TOOL_TRACE, ArtifactKind.RESULT} +) + + +def is_reserved_artifact_kind(kind: ArtifactKind) -> bool: + """"the budget is partitioned by reservation: `build` + `transcript` + `tool_trace` + `result` + are reserved (always admitted); recordings next; `trace`/`log`/`other` last.\"""" + return kind in _RESERVED_ARTIFACT_KINDS + + +_RECORDING_ARTIFACT_KINDS = frozenset( + { + ArtifactKind.RECORDING_COMBINED, + ArtifactKind.RECORDING_STEREO, + ArtifactKind.RECORDING_CUSTOMER, + ArtifactKind.RECORDING_ASSISTANT, + } +) + + +def priority_class(kind: ArtifactKind) -> int: + """N16: the contract's three-tier budget partition as a total order, lower = admitted first -- + `0` reserved (`is_reserved_artifact_kind`, always admitted), `1` recordings, `2` `trace`/`log`/ + `other` (admitted last).""" + if is_reserved_artifact_kind(kind): + return 0 + if kind in _RECORDING_ARTIFACT_KINDS: + return 1 + return 2 + + +class ArtifactBudgetTracker: + """Client-side mirror of "budget = upload admission ... the guest enforces it first": a + per-job cumulative cap across attempts, deduplicated by digest. `would_admit` is a pure check a + caller makes before calling `ArtifactsClient.upload` for a non-reserved kind; when it returns + `False` the upload is skipped (and named in a `log` event -- the caller's job, not this + tracker's). This class does not sequence "refused newest-first" itself -- it has no visibility + into candidate ordering across scenarios, which only the scheduler (P9) has; it supplies the + admission arithmetic that policy is built on. + + `recording_headroom_bytes` (N16, default 0 -- no behavior change unless a caller opts in): + bytes of the remaining budget reserved for recordings not yet seen, subtracted from what a + `trace`/`log`/`other` (priority class 2) candidate is allowed to consume. This tracker has no + visibility into how many recording bytes are still coming (only the scheduler does), so it + cannot give a perfect answer -- reserving a caller-supplied headroom is the honest, testable + subset of "recordings next; trace/log/other last" this class alone can enforce. + """ + + def __init__(self, max_artifact_bytes: int, *, recording_headroom_bytes: int = 0) -> None: + self._max_bytes = max_artifact_bytes + self._admitted_bytes = 0 + self._seen_digests: set[str] = set() + self._recording_headroom_bytes = recording_headroom_bytes + + def would_admit(self, kind: ArtifactKind, size: int, *, digest: str) -> bool: + if digest in self._seen_digests: + return True # already counted; a duplicate upload never grows the budget further + tier = priority_class(kind) + if tier == 0: + return True + remaining = self._max_bytes - self._admitted_bytes + if tier == 2: + remaining -= self._recording_headroom_bytes + return size <= remaining + + def record(self, kind: ArtifactKind, size: int, *, digest: str) -> None: + del kind # reservation already resolved by would_admit; recorded uniformly here + if digest in self._seen_digests: + return + self._seen_digests.add(digest) + self._admitted_bytes += size + + @property + def admitted_bytes(self) -> int: + return self._admitted_bytes + + +class ArtifactManifestEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + + artifact_id: str + kind: ArtifactKind + size: int = Field(ge=0) + scenario_key: str | None = None + + @model_validator(mode="after") + def _validate(self) -> "ArtifactManifestEntry": + if not is_valid_digest(self.artifact_id): + raise ValueError(f"artifact_id_invalid: {self.artifact_id!r}") + return self + + +class ArtifactManifestDraft(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str = MANIFEST_SCHEMA_VERSION + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + entries: list[ArtifactManifestEntry] + complete: bool + digest: str + + @model_validator(mode="after") + def _validate(self) -> "ArtifactManifestDraft": + if self.schema_version != MANIFEST_SCHEMA_VERSION: + raise ValueError(f"manifest_schema_unsupported: {self.schema_version}") + if not is_valid_digest(self.digest): + raise ValueError(f"manifest_digest_invalid: {self.digest!r}") + expected = whole_object_digest(self.model_dump(mode="json", exclude={"digest"})) + if self.digest != expected: + unset = _unset_default_fields(self) + hint = f" -- fields not explicitly set, filled from defaults: {', '.join(unset)}" if unset else "" + raise ValueError(f"manifest_digest_mismatch{hint}") + return self + + +def build_artifact_manifest( + *, + job_id: str, + attempt_id: str, + attempt_number: int, + entries: list[dict[str, Any]], + complete: bool, +) -> dict[str, Any]: + """Same pattern as `build_result_receipt`/`build_event_record`: digest computed on the raw + input, then re-verified by the model that consumes it.""" + core: dict[str, Any] = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "job_id": job_id, + "attempt_id": attempt_id, + "attempt_number": attempt_number, + "entries": entries, + "complete": complete, + } + digest = whole_object_digest(core) + draft = ArtifactManifestDraft.model_validate({**core, "digest": digest}) + return draft.model_dump(mode="json") + + +@dataclass(frozen=True) +class ArtifactUploadResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +@dataclass(frozen=True) +class ManifestPushResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +_DEFAULT_ARTIFACT_CONTENT_TYPES: dict[ArtifactKind, str] = { + ArtifactKind.RECORDING_COMBINED: "video/mp4", + ArtifactKind.RECORDING_STEREO: "video/mp4", + ArtifactKind.RECORDING_CUSTOMER: "video/mp4", + ArtifactKind.RECORDING_ASSISTANT: "video/mp4", + ArtifactKind.TRANSCRIPT: "application/json", + ArtifactKind.RESULT: "application/json", +} + + +def _default_content_type(kind: ArtifactKind) -> str: + """N17: §3a pins recordings to mp4 and `transcript` to a JSON array; a platform serving these + back to a UI needs an accurate `Content-Type`, not a blanket octet-stream.""" + return _DEFAULT_ARTIFACT_CONTENT_TYPES.get(kind, "application/octet-stream") + + +class ArtifactsClient: + """Content-addressed upload (§3a) + manifest delivery (§3b) to `endpoints.artifacts`. + + `upload` verifies the given bytes actually hash to the claimed `artifact_id` BEFORE ever + calling the transport -- a local, zero-cost check that catches a caller bug (wrong id, wrong + bytes) without spending a round trip on it; the platform's own `422 digest_mismatch` remains + the authority for anything this local check cannot see (partial reads, transport corruption). + On a `422 digest_mismatch` FROM THE PLATFORM specifically, the contract grants exactly one + extra whole-upload retry ("re-upload once, then the referencing scenario is `errored`") -- + distinct from `_perform_with_retry`'s own loop, which treats every 422 as `PERMANENT_ITEM` and + never retries it; this method wraps that loop in one more, narrower retry layer that fires only + for that one code. + + Size accounting: `X-Artifact-Size` is never a caller-supplied value -- it is always derived + from `len(data)`, the same bytes actually transmitted, so a `422 size_mismatch` against what + this client sends is structurally unreachable from here (the platform's own count remains the + authority for what actually arrived over the wire). + + N18: once a 413 `artifact_budget_exceeded` is observed, this instance latches locally -- every + later `upload()` for a NON-reserved kind is refused without contacting the platform at all + ("stop uploading non-reserved kinds, log, continue the run"); reserved kinds keep uploading + (they are always admitted, budget or not). + """ + + def __init__( + self, + capabilities: HostedCapabilities, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + chunk_threshold_bytes: int = ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES, + chunk_size_bytes: int = 8 * 1024 * 1024, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._chunk_threshold_bytes = chunk_threshold_bytes + self._chunk_size_bytes = chunk_size_bytes + self._channel_state = channel_state or ChannelState() + self._budget_exhausted = False # N18 + + def upload( + self, + artifact_id_hex: str, + data: bytes, + *, + kind: ArtifactKind, + scenario_key: str | None = None, + content_type: str | None = None, + deadline: float | None = None, + ) -> ArtifactUploadResult: + self._channel_state.check() + if self._budget_exhausted and not is_reserved_artifact_kind(kind): + logger.warning( + "artifacts upload: budget already exhausted (413 artifact_budget_exceeded observed " + "earlier this attempt) -- skipping non-reserved kind=%s without contacting the " + "platform", kind.value, + ) + return ArtifactUploadResult( + delivered=False, + already_existed=False, + error=ChannelError( + ChannelOutcome.BUDGET_EXCEEDED, None, "artifact_budget_exceeded", + "budget already exhausted for this attempt (latched locally)", status_code=None, + ), + ) + if not re.fullmatch(r"[0-9a-f]{64}", artifact_id_hex): + raise ValueError(f"artifact_id_invalid: {artifact_id_hex!r}") + if artifact_id_hex == "manifest": + # Unreachable through the hex check above (`manifest` isn't 64 hex chars) but the + # contract calls this collision out by name ("`manifest` is a reserved segment"). + raise ValueError("artifact_id_reserved: manifest") + computed = hashlib.sha256(data).hexdigest() + if computed != artifact_id_hex: + raise ValueError( + f"artifact_digest_mismatch_local: expected {artifact_id_hex}, computed {computed}" + ) + + url = f"{self._capabilities.endpoints.artifacts}{artifact_id_hex}/" + size = len(data) + headers = { + **self._capabilities.auth_headers(), + "X-Artifact-Kind": kind.value, + "X-Artifact-Size": str(size), + "Content-Type": content_type or _default_content_type(kind), + } + if scenario_key is not None: + headers["X-Scenario-Key"] = scenario_key + + def perform(_attempt: int) -> TransportResponse: + # A fresh generator per attempt when chunked -- see `_iter_chunks`. + body: bytes | Iterator[bytes] = ( + _iter_chunks(data, self._chunk_size_bytes) + if size > self._chunk_threshold_bytes + else data + ) + return self._transport.request("PUT", url, headers=headers, data=body) + + response: TransportResponse | None = None + error: ChannelError | None = None + for outer_attempt in range(2): # "re-upload once" on a platform-confirmed digest mismatch + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if not (error is not None and error.code == "digest_mismatch" and outer_attempt == 0): + break + + if error is not None and error.outcome is ChannelOutcome.BUDGET_EXCEEDED: + self._budget_exhausted = True # N18 + + if error is not None or response is None: + return ArtifactUploadResult(delivered=False, already_existed=False, error=error) + # N20: `200` is read as "already exists" per the contract's content-addressed upload + # semantics; the success code for a genuinely NEW upload is `201` (this module's own + # `FakePlatform` matches that but it is unconfirmed against the real platform). + return ArtifactUploadResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + def push_manifest(self, manifest: dict[str, Any], *, deadline: float | None = None) -> ManifestPushResult: + self._channel_state.check() + url = f"{self._capabilities.endpoints.artifacts}manifest/" + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", url, headers=self._capabilities.auth_headers(), json_body=manifest + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None and error.code == "attempt_superseded": # N22 + self._channel_state.latch(HostedAttemptSupersededError(error)) + if error is not None or response is None: + return ManifestPushResult(delivered=False, already_existed=False, error=error) + # N20: same caveat as receipts/uploads above -- `200` == duplicate is contract-stated, + # the new-manifest success code is `201` per `FakePlatform`, unconfirmed against the real + # platform. + return ManifestPushResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + +__all__ = [ + "ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES", + "CAPABILITIES_PATH", + "CAPABILITIES_SCHEMA_VERSION", + "EVENTS_MAX_BATCH", + "EVENTS_MAX_BATCH_BYTES", + "EVENT_PAYLOAD_MAX_BYTES", + "EVENT_SCHEMA_VERSION", + "FLUSH_WINDOW_SECONDS", + "MANIFEST_SCHEMA_VERSION", + "RESULT_SCHEMA_VERSION", + "ArtifactBudgetTracker", + "ArtifactKind", + "ArtifactManifestDraft", + "ArtifactManifestEntry", + "ArtifactUploadResult", + "ArtifactsClient", + "BaselineFrozenPayload", + "BaselineInputsChangedPayload", + "CallSummary", + "CapabilitiesError", + "ChannelError", + "ChannelOutcome", + "ChannelState", + "CheckpointEvaluation", + "DegradeReason", + "EvaluationResult", + "EventsClient", + "EventsFlushResult", + "HostedAttemptSupersededError", + "HostedCapabilities", + "HostedChannelFailedError", + "HostedEndpoints", + "HostedEvent", + "HostedEventDraft", + "HostedFencedError", + "LogLevel", + "LogPayload", + "ManifestPushResult", + "MetricEvaluation", + "OutboundError", + "OutboundEventType", + "OutboundSpool", + "OutboundSpoolError", + "ParallelismDegradedPayload", + "ReceiptPushResult", + "RequestsTransport", + "ResultReceiptDraft", + "ResultsClient", + "RetryPolicy", + "ScenarioCounts", + "ScenarioRetriedPayload", + "ScenarioStartedPayload", + "ScenarioStatus", + "SpooledRecord", + "StageChangedPayload", + "SubGoalResult", + "TerminalFailure", + "TerminalPayload", + "TerminalReason", + "Transport", + "TransportError", + "TransportResponse", + "WorldUnhealthyPayload", + "build_artifact_manifest", + "build_event_record", + "build_result_receipt", + "build_skipped_receipt", + "canonical_bytes", + "classify_response", + "compute_backoff_seconds", + "event_payload_digest", + "format_rfc3339_millis", + "is_reserved_artifact_kind", + "is_valid_digest", + "is_valid_rfc3339_millis", + "load_capabilities", + "priority_class", + "redact_outbound_text", + "sha256_digest", + "truncate_log_message", + "whole_object_digest", +] diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py new file mode 100644 index 00000000..ba734b4a --- /dev/null +++ b/tests/harness/test_outbound.py @@ -0,0 +1,2959 @@ +"""Outbound reporting, per `outbound-channels.md` v1.2. Two parts, matching `outbound.py`: + +Foundations (part 1): + +- Canonicalization: `canonical_bytes`/`event_payload_digest`/`whole_object_digest` against fixed + expected digests, independently derived by running the contract's own literal + ``json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False)`` + sha256 call in + a bare Python shell -- not by calling back into this module -- so the vectors don't just prove + the implementation agrees with itself. +- Capabilities: `HostedCapabilities`/`load_capabilities` against the contract's own example, + its endpoint-trailing-slash rule, schema-version gate, and the load-then-unlink lifetime rule. +- Events + spool: `build_event_record`/`HostedEvent` shape validation for all nine closed event + types, and `OutboundSpool`'s contiguous-from-1 sequencing, its crash/torn-tail recovery rule + (a synthetic partial write, simulating a crash mid-append), and watermark semantics. + +Transport (part 2): + +- Pure functions: the full `classify_response` error map (every status the contract's "Error + responses"/"Failure semantics summary" names, plus the unlisted-4xx catch-all), the full-jitter + `compute_backoff_seconds` formula, `format_rfc3339_millis`, and `ArtifactBudgetTracker`. +- Typed models: `ResultReceiptDraft` (contract-shaped, the `skipped`/`errored` exact-shape rules, + digest tamper rejection) and `ArtifactManifestDraft` (digest scope, `complete` distinguishing + documents). +- `FakePlatform`: an in-process, no-sockets `Transport` implementation with real server-side state + (accepted events + watermark, receipts keyed by `(job_id, scenario_key)`, artifacts keyed by + digest, manifests keyed by `(attempt_id, digest)`) that `EventsClient`/`ResultsClient`/ + `ArtifactsClient` are driven against for: success + watermark advance, transient-then-success + (a queued `TransportError` then a queued 5xx then real success), 429 honoring `Retry-After` over + the computed backoff, deterministic rejection (never retried, at both the per-event and + whole-request level), `FENCED`/`CHANNEL_FAILED` raising the two channel-ending exceptions, the + artifact digest-mismatch "re-upload once" rule, budget refusal, chunked-transfer reassembly, and + crash-between-send-and-ack redelivery (the platform durably records a write and then the + response is lost -- `FakePlatform.crash_after_next_write` -- proving a redelivery of the same + record is a safe no-op, both within one client call via its own retry loop and across two + independent calls simulating a real process restart). +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import logging +import os +import threading +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from fi.alk.harness.job import FailureDomain, HarnessStage +from fi.alk.harness.outbound import ( + CAPABILITIES_SCHEMA_VERSION, + EVENT_PAYLOAD_MAX_BYTES, + ArtifactBudgetTracker, + ArtifactKind, + ArtifactManifestDraft, + ArtifactsClient, + CapabilitiesError, + ChannelOutcome, + ChannelState, + EventsClient, + HostedAttemptSupersededError, + HostedCapabilities, + HostedChannelFailedError, + HostedEndpoints, + HostedEvent, + HostedEventDraft, + HostedFencedError, + OutboundError, + OutboundEventType, + OutboundSpool, + OutboundSpoolError, + ResultReceiptDraft, + ResultsClient, + RetryPolicy, + ScenarioStatus, + StageChangedPayload, + TransportError, + TransportResponse, + build_artifact_manifest, + build_event_record, + build_result_receipt, + build_skipped_receipt, + canonical_bytes, + classify_response, + compute_backoff_seconds, + event_payload_digest, + format_rfc3339_millis, + is_valid_digest, + is_valid_rfc3339_millis, + load_capabilities, + priority_class, + redact_outbound_text, + sha256_digest, + truncate_log_message, + whole_object_digest, +) + +NOW = datetime(2026, 8, 25, 10, 14, 3, 412000, tzinfo=timezone.utc) +LATER = datetime(2026, 8, 25, 10, 17, 7, 623000, tzinfo=timezone.utc) + + +@pytest.fixture(autouse=True) +def _clear_spool_registry() -> Any: + """M6's in-process registry caches `OutboundSpool` instances (and their open flock fds) for the + life of the interpreter, keyed on `(resolved_root, name)`. Different tests use different + `tmp_path` roots so there is no cross-test key collision, but without this the fds would still + accumulate across the whole session. Runs after every test.""" + yield + OutboundSpool._clear_registry_for_tests() + + +# --- Canonicalization ------------------------------------------------------------------------- + + +def test_canonical_bytes_sorts_keys_and_uses_compact_separators() -> None: + assert canonical_bytes({"b": 1, "a": 2}) == b'{"a":2,"b":1}' + + +def test_canonical_bytes_does_not_escape_non_ascii() -> None: + # ensure_ascii=False: a non-ASCII character is emitted as its raw UTF-8 bytes, not a \uXXXX + # escape -- the contract is explicit about the exact `json.dumps` call, and this is the one + # kwarg that differs from Python's default. + encoded = canonical_bytes({"x": "…"}) + assert encoded == b'{"x":"\xe2\x80\xa6"}' + assert b"\\u" not in encoded + + +def test_canonical_bytes_absent_and_null_are_different_bytes() -> None: + assert canonical_bytes({}) != canonical_bytes({"x": None}) + assert canonical_bytes({"x": None}) == b'{"x":null}' + + +def test_event_payload_digest_matches_independently_computed_vector() -> None: + # The Channel 1 example payload from outbound-channels.md, transcribed verbatim (including its + # literal "…" ellipsis character, which also exercises the ensure_ascii=False rule end-to-end). + payload = {"scenario_key": "…", "world_index": 2, "scenario_attempt": 1} + digest = event_payload_digest(payload) + # Computed independently via `python -c "import json,hashlib; ..."` against the contract's own + # literal algorithm text, not by calling back into this module. + assert digest == ( + "sha256:7a8037965edd1f5391e6d59d0556313938abbd272ca162cbf75aa1510ca7e0f1" + ) + assert is_valid_digest(digest) + + +def test_event_payload_digest_scope_is_payload_only_not_the_envelope() -> None: + payload = {"world_index": 2, "cause": "boom"} + envelope = {"event_id": "event_x", "stage": "running", "type": "world_unhealthy", "payload": payload} + # The envelope's other fields must not affect the digest -- only `payload` is in scope. + assert event_payload_digest(payload) == event_payload_digest(envelope["payload"]) + assert sha256_digest(canonical_bytes(payload)) == event_payload_digest(payload) + + +def test_whole_object_digest_scope_excludes_the_digest_key_by_popping_it() -> None: + core = {"a": 1, "b": 2} + with_digest = {"a": 1, "digest": "sha256:" + "f" * 64, "b": 2} + # "the whole object with the `digest` key absent" -- popped, so the digest's own value (right + # or wrong, present or not) can never affect the hash the platform re-derives and checks. + assert whole_object_digest(core) == whole_object_digest(with_digest) + + +def test_whole_object_digest_matches_receipt_and_manifest_shaped_examples() -> None: + # Channel 2's example receipt object, minus its own `digest` field (as if the guest is about + # to compute it) -- proves the generic function against a receipt-shaped whole object, not + # just the toy `{a, b}` case above. + receipt = { + "schema_version": "futureagi.harness-result.v1", + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "scenario_key": "suspended-account-blocked", + "scenario_id": "sid-1", + "scenario_attempt": 1, + "world_index": 2, + "status": "passed", + "sub_goals": [{"name": "n", "held": True, "reason": None, "judged": False}], + "evaluations": [], + "call": None, + "failure": None, + } + digest = whole_object_digest(receipt) + assert is_valid_digest(digest) + # FIXED expected hex (ranked missing test 2): computed independently via + # `json.dumps(receipt, sort_keys=True, separators=(",", ":"), ensure_ascii=False, + # allow_nan=False)` + sha256 in a bare Python shell against the contract's own canonicalization + # clause -- not by calling back into this module. This is the digest the platform ACTUALLY + # verifies (event digests are only MAY-verified; receipt/manifest digests are verified), so a + # self-consistency-only test (is_valid_digest, invariance to `digest`, inequality between two + # objects) cannot catch a canonicalization regression that both this test and the module agree + # on by construction -- a fixed vector can. + assert digest == "sha256:b1492bcb2f62c3d79dc4d0d1b3f5097a3784112fc00276a2dc483297375652af" + # Adding a `digest` key of any value must not change the result (scope excludes it). + assert whole_object_digest({**receipt, "digest": "sha256:" + "0" * 64}) == digest + + # Channel 3b's manifest example, minus `digest`. + manifest = { + "schema_version": "futureagi.harness-manifest.v1", + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "entries": [{"artifact_id": "sha256:" + "1" * 64, "kind": "result", "size": 10, "scenario_key": "k"}], + "complete": True, + } + manifest_digest = whole_object_digest(manifest) + assert is_valid_digest(manifest_digest) + # FIXED expected hex, same independent-derivation method as above. + assert manifest_digest == "sha256:f18786749321d103cf30ebe11e935bf9177d5297783ee2dec00cb20ad8dae13d" + assert manifest_digest != digest + + +def test_canonical_bytes_rejects_nan() -> None: + # v1.3: allow_nan=False -- NaN/Infinity are not RFC 8259 and must fail typed at the emitter + # rather than produce bytes no strict parser downstream can read. + with pytest.raises(OutboundError) as excinfo: + canonical_bytes({"score": float("nan")}) + assert excinfo.value.code == "canonical_value_not_finite" + + +@pytest.mark.parametrize("value", [float("inf"), float("-inf")]) +def test_canonical_bytes_rejects_infinity(value: float) -> None: + with pytest.raises(OutboundError): + canonical_bytes({"score": value}) + + +def test_canonical_bytes_is_still_byte_identical_for_valid_input_after_allow_nan_false() -> None: + # "for valid input the bytes are identical" (v1.3) -- allow_nan=False changes nothing about + # ordinary values. + assert canonical_bytes({"b": 1, "a": 2}) == b'{"a":2,"b":1}' + + +def test_whole_object_digest_rejects_a_non_json_native_value_naming_the_key_path() -> None: + with pytest.raises(OutboundError) as excinfo: + whole_object_digest({"a": {"b": datetime(2026, 1, 1, tzinfo=timezone.utc)}}) + assert excinfo.value.code == "digest_value_not_json_native" + assert "a.b" in excinfo.value.message + + +def test_whole_object_digest_rejects_a_non_string_dict_key() -> None: + with pytest.raises(OutboundError): + whole_object_digest({"a": {1: "x"}}) + + +# --- Capabilities ------------------------------------------------------------------------------ + +_ATTEMPT_ID = "22222222-2222-2222-2222-222222222222" + +VALID_CAPABILITIES = { + "schema_version": CAPABILITIES_SCHEMA_VERSION, + "job_id": "11111111-1111-1111-1111-111111111111", + "attempt_id": _ATTEMPT_ID, + "attempt_number": 1, + "fence": "opaque-fence", + "expires_at": "2026-08-25T12:00:00.000Z", + "token": "bearer-token", + "endpoints": { + "events": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/events/", + "results": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/results/", + "artifacts": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/artifacts/", + "scenarios": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/scenarios/", + }, +} + + +def _write(path: Path, value: dict) -> Path: + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +def test_load_capabilities_parses_the_contract_example_and_builds_auth_headers(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + assert capabilities.endpoints.events.endswith("/") + assert capabilities.auth_headers() == { + "Authorization": "Bearer bearer-token", + "X-Harness-Fence": "opaque-fence", + } + + +def test_load_capabilities_unlinks_the_file_after_a_successful_load(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + load_capabilities(path) # default unlink=True + assert not path.exists() + + +def test_load_capabilities_does_not_unlink_on_a_validation_failure(tmp_path: Path) -> None: + bad = {**VALID_CAPABILITIES, "schema_version": "futureagi.harness-capabilities.v0"} + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError): + load_capabilities(path) + # A crash mid-load must never destroy the only copy of a not-yet-consumed bearer. + assert path.exists() + + +def test_load_capabilities_rejects_missing_file(tmp_path: Path) -> None: + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(tmp_path / "missing.json") + assert excinfo.value.code == "capabilities_file_missing" + + +def test_load_capabilities_rejects_unreadable_file(tmp_path: Path) -> None: + if os.geteuid() == 0: + pytest.skip("permission bits don't block a root reader") + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + path.chmod(0o000) + try: + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_unreadable" + finally: + path.chmod(0o600) + + +def test_load_capabilities_rejects_malformed_json(tmp_path: Path) -> None: + path = tmp_path / "capabilities.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_malformed" + + +def test_load_capabilities_rejects_valid_json_that_is_not_an_object(tmp_path: Path) -> None: + path = tmp_path / "capabilities.json" + path.write_text("[1, 2, 3]", encoding="utf-8") + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_malformed" + + +def test_load_capabilities_rejects_wrong_schema_version(tmp_path: Path) -> None: + bad = {**VALID_CAPABILITIES, "schema_version": "futureagi.harness-capabilities.v0"} + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + # Lens 5: this code used to be unreachable -- pydantic wrapped it into a ValidationError that + # only ever surfaced as the generic capabilities_field_invalid. + assert excinfo.value.code == "capabilities_schema_unsupported" + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_an_endpoint_missing_its_trailing_slash( + tmp_path: Path, channel: str +) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].rstrip("/") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + # Also previously unreachable as a code -- same dead-code shape as the schema check above. + assert excinfo.value.code == "capabilities_endpoint_invalid" + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_a_non_https_endpoint(tmp_path: Path, channel: str) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].replace("https://", "http://") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_endpoint_insecure" + + +def test_load_capabilities_rejects_an_expired_token(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path, now=lambda: datetime(2027, 1, 1, tzinfo=timezone.utc)) + assert excinfo.value.code == "capabilities_expired" + assert path.exists() # a failed load must never unlink -- same rule as any other rejection + + +def test_load_capabilities_accepts_a_token_not_yet_expired(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, now=lambda: datetime(2020, 1, 1, tzinfo=timezone.utc)) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_an_endpoint_whose_attempt_id_segment_disagrees( + tmp_path: Path, channel: str +) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].replace(_ATTEMPT_ID, "some-other-attempt") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_attempt_mismatch" + + +def test_load_capabilities_field_invalid_message_never_echoes_the_input_value(tmp_path: Path) -> None: + # MIN-3: the file carries the bearer -- a validation message built from pydantic's default + # str(exc) would embed the failing field's raw value. + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["token"] = 987654321 + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_field_invalid" + assert "987654321" not in excinfo.value.message + + +def test_load_capabilities_unlink_failure_is_reported_via_the_callback(tmp_path: Path) -> None: + # MIN-7: previously swallowed unconditionally; now the caller can tell. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + original_unlink = Path.unlink + + def failing_unlink(self: Path, *a: Any, **kw: Any) -> None: + if self == path: + raise OSError("simulated unlink failure") + return original_unlink(self, *a, **kw) + + reported: list[OSError] = [] + Path.unlink = failing_unlink # type: ignore[method-assign] + try: + load_capabilities(path, on_unlink_failure=reported.append) + finally: + Path.unlink = original_unlink # type: ignore[method-assign] + assert len(reported) == 1 + assert path.exists() # the failed unlink really did leave the file behind + + +def test_load_capabilities_insecure_file_mode_warns_but_does_not_block_the_load( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # MIN-5: fail-safe -- a wrong mode/owner is a loud warning, never a rejection. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + path.chmod(0o644) + with caplog.at_level("WARNING"): + capabilities = load_capabilities(path, unlink=False) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + assert any("capabilities file mode" in record.message for record in caplog.records) + + +def test_hosted_endpoints_rejects_unknown_field() -> None: + with pytest.raises(ValidationError): + HostedCapabilities.model_validate({**VALID_CAPABILITIES, "unexpected": "field"}) + + +def test_hosted_endpoints_shape_validator_still_fires_on_direct_construction() -> None: + # Defense-in-depth: constructing the model directly (bypassing load_capabilities' + # pre-pydantic checks entirely) must still reject a malformed endpoint. + with pytest.raises(ValidationError): + HostedEndpoints.model_validate( + {"events": "http://x/", "results": "https://x/", "artifacts": "https://x/", "scenarios": "https://x/"} + ) + + +def test_capabilities_expires_at_is_serialized_as_millis_z() -> None: + capabilities = HostedCapabilities.model_validate( + {**VALID_CAPABILITIES, "expires_at": "2026-08-25T12:00:00.123456+00:00"} + ) + assert capabilities.model_dump(mode="json")["expires_at"] == "2026-08-25T12:00:00.123Z" + + +def test_capabilities_event_builder_binds_identity_fields_from_the_token(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + build = capabilities.event_builder() + record = build( + event_id="event_" + "a" * 32, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + assert record["job_id"] == capabilities.job_id + assert record["attempt_id"] == capabilities.attempt_id + assert record["attempt_number"] == capabilities.attempt_number + + +def test_capabilities_event_builder_forwards_extra_secret_values_to_log_message(tmp_path: Path) -> None: + # P4: `event_builder()` had no way to receive the job's declared secret VALUES, so a caller + # wired for it (the shipped adapter binds identity via `event_builder()` and routes every event + # through it) could never satisfy N9's redaction requirement for `log.message`/ + # `terminal.failure.message` through this API -- only the userinfo scrub reached those fields. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + build = capabilities.event_builder(extra_secret_values=("s3cr3t-value",)) + record = build( + event_id="event_" + "a" * 32, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "error", "message": "build failed: s3cr3t-value leaked in the output"}, + ) + assert "s3cr3t-value" not in record["payload"]["message"] + assert "***" in record["payload"]["message"] + + +# --- Events -------------------------------------------------------------------------------- + + +def _event(type_: OutboundEventType, stage: HarnessStage, payload: dict) -> dict: + return build_event_record( + event_id="event_" + "a" * 32, + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=stage, + type=type_, + payload=payload, + ) + + +def test_build_event_record_has_no_sequence_key_and_embeds_a_valid_digest() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "hello"} + ) + assert "sequence" not in record + assert is_valid_digest(record["digest"]) + assert record["digest"] == event_payload_digest(record["payload"]) + + +def test_hosted_event_requires_sequence_but_draft_does_not() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "debug", "message": "m"} + ) + HostedEventDraft.model_validate(record) # no sequence needed + with pytest.raises(ValidationError): + HostedEvent.model_validate(record) # sequence is required on the full wire object + HostedEvent.model_validate({**record, "sequence": 1}) + + +@pytest.mark.parametrize( + ("type_", "stage", "payload"), + [ + ( + OutboundEventType.STAGE_CHANGED, + HarnessStage.RUNNING, + {"from": "connecting_agent", "to": "running"}, + ), + (OutboundEventType.STAGE_CHANGED, HarnessStage.QUEUED, {"from": None, "to": "queued"}), + ( + OutboundEventType.PARALLELISM_DEGRADED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"requested": 4, "effective": 1, "reason": "conformance_gate_failed"}, + ), + ( + OutboundEventType.BASELINE_FROZEN, + HarnessStage.VALIDATING_ENVIRONMENT, + {"inputs_digest": "sha256:" + "1" * 64, "baseline_ref": "build-1"}, + ), + ( + OutboundEventType.BASELINE_INPUTS_CHANGED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"previous_digest": None, "current_digest": "sha256:" + "2" * 64}, + ), + ( + OutboundEventType.WORLD_UNHEALTHY, + HarnessStage.RUNNING, + {"world_index": 1, "cause": "sentinel failed"}, + ), + ( + OutboundEventType.SCENARIO_STARTED, + HarnessStage.RUNNING, + {"scenario_key": "k", "world_index": 0, "scenario_attempt": 1}, + ), + ( + OutboundEventType.SCENARIO_RETRIED, + HarnessStage.RUNNING, + {"scenario_key": "k", "from_world": 0, "to_world": 1}, + ), + (OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "warning", "message": "m"}), + ( + OutboundEventType.TERMINAL, + HarnessStage.COMPLETED, + { + "stage": "completed", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 2, "failed": 0, "errored": 0, "skipped": 0}, + }, + ), + ( + OutboundEventType.TERMINAL, + HarnessStage.CANCELED, + { + "stage": "canceled", + "reason": "user_canceled", + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 1}, + }, + ), + ], +) +def test_every_closed_event_type_accepts_its_contract_shaped_payload( + type_: OutboundEventType, stage: HarnessStage, payload: dict +) -> None: + record = _event(type_, stage, payload) + HostedEvent.model_validate({**record, "sequence": 1}) + + +def test_stage_changed_to_must_equal_the_events_own_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.STAGE_CHANGED, + HarnessStage.RUNNING, + {"from": None, "to": "grading"}, + ) + + +def test_stage_changed_rejects_the_python_attribute_name_instead_of_the_wire_alias() -> None: + # M9: populate_by_name=True previously let a caller who writes `from_stage` (the natural + # mistake -- that's the attribute name) pass validation while spooling an undefined wire key. + with pytest.raises(ValidationError): + StageChangedPayload.model_validate({"from_stage": "queued", "to": "running"}) + + +def test_terminal_payload_stage_must_equal_the_events_own_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.TERMINAL, + HarnessStage.COMPLETED, + { + "stage": "failed", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + + +def test_terminal_payload_rejects_a_non_terminal_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.TERMINAL, + HarnessStage.RUNNING, + { + "stage": "running", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + + +def test_scenario_started_scenario_attempt_is_closed_to_one_or_two() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.SCENARIO_STARTED, + HarnessStage.RUNNING, + {"scenario_key": "k", "world_index": 0, "scenario_attempt": 3}, + ) + + +def test_parallelism_degraded_effective_must_be_strictly_below_requested() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.PARALLELISM_DEGRADED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"requested": 3, "effective": 3, "reason": "fixed_port"}, + ) + + +def test_unknown_event_type_is_rejected_by_the_closed_vocabulary() -> None: + with pytest.raises(ValidationError): + build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type="not_a_real_type", # type: ignore[arg-type] + payload={}, + ) + + +def test_payload_with_an_unknown_key_is_rejected() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.LOG, + HarnessStage.RUNNING, + {"level": "info", "message": "m", "extra": "nope"}, + ) + + +def test_oversized_payload_for_a_non_log_type_is_still_hard_rejected() -> None: + # M8: only `log` is truncated instead of rejected -- every other type keeps the hard rejection. + # `baseline_frozen` has no field-level max_length, so an artificially huge `baseline_ref` is a + # clean way to exercise the envelope-level size check for a non-log type. + with pytest.raises(ValidationError): + _event( + OutboundEventType.BASELINE_FROZEN, + HarnessStage.VALIDATING_ENVIRONMENT, + {"inputs_digest": "sha256:" + "1" * 64, "baseline_ref": "x" * (EVENT_PAYLOAD_MAX_BYTES + 1)}, + ) + + +def test_log_payload_is_truncated_rather_than_rejected_when_oversized() -> None: + huge_message = "x" * (EVENT_PAYLOAD_MAX_BYTES * 2) + record = _event(OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "error", "message": huge_message}) + assert len(canonical_bytes(record["payload"])) <= EVENT_PAYLOAD_MAX_BYTES + assert record["payload"]["message"].endswith("…[truncated]") + # The digest must match the TRUNCATED bytes actually spooled, not the original message. + assert record["digest"] == event_payload_digest(record["payload"]) + HostedEvent.model_validate({**record, "sequence": 1}) # round-trips cleanly + + +def test_truncate_log_message_is_a_no_op_when_already_within_budget() -> None: + assert truncate_log_message("info", "short message") == "short message" + + +def test_truncate_log_message_fits_the_exact_byte_budget() -> None: + message = truncate_log_message("error", "y" * 1000, max_payload_bytes=64) + assert message.endswith("…[truncated]") + assert len(canonical_bytes({"level": "error", "message": message})) <= 64 + + +def test_a_tampered_digest_is_rejected() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "m"} + ) + record["digest"] = "sha256:" + "0" * 64 + with pytest.raises(ValidationError): + HostedEventDraft.model_validate(record) + + +def test_event_id_is_checked_against_utf8_byte_length_not_just_character_count() -> None: + # N-2: max_length=64 on the model counts characters; the platform's column is presumably + # bytes. 64 non-ASCII characters can exceed 64 bytes while passing the character-count check. + event_id = "€" * 64 # 64 chars, 3 bytes each in UTF-8 => 192 bytes + with pytest.raises(ValidationError): + build_event_record( + event_id=event_id, + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + + +def test_hosted_event_emitted_at_is_serialized_as_millis_z_not_the_pydantic_default() -> None: + record = _event(OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "m"}) + assert record["emitted_at"] == "2026-08-25T10:14:03.412Z" + + +def test_hosted_event_emitted_at_rejects_a_naive_datetime() -> None: + with pytest.raises(ValidationError): + build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=datetime(2026, 8, 25, 10, 0, 0), # naive + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + + +def test_hosted_event_emitted_at_normalizes_a_non_utc_offset_to_z() -> None: + ist = datetime(2026, 8, 25, 15, 44, 3, 412000, tzinfo=timezone(timedelta(hours=5, minutes=30))) + record = build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=ist, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + assert record["emitted_at"] == "2026-08-25T10:14:03.412Z" + + +# --- Spool --------------------------------------------------------------------------------- + + +def test_spool_assigns_contiguous_sequence_numbers_starting_at_one(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + sequences = [spool.append({"event_id": f"e{i}"}).sequence for i in range(5)] + assert sequences == [1, 2, 3, 4, 5] + assert spool.next_sequence == 6 + + +def test_spool_appended_body_round_trips_the_assigned_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + record = spool.append({"event_id": "e1", "payload": {"x": 1}}) + decoded = record.decode() + assert decoded["sequence"] == 1 + assert decoded["event_id"] == "e1" + + +def test_spool_recovers_next_sequence_across_a_clean_restart(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(3): + first.append({"event_id": f"e{i}"}) + + # M6: the same (root, name) returns the CACHED instance within one process -- simulating a + # real restart (a fresh interpreter, an empty registry) needs an explicit evict. + OutboundSpool._forget_for_tests(tmp_path, "events") + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second is not first + assert second.next_sequence == 4 + assert second.append({"event_id": "e3"}).sequence == 4 + + +def test_spool_recovers_from_a_torn_tail_by_truncating_and_reusing_the_sequence( + tmp_path: Path, +) -> None: + """Simulates a crash mid-write: a partial line with no trailing newline is appended directly + to the spool file (bypassing `append`, the way a killed process would leave the file). The + recovery rule this module documents says the next spool instance must detect this, truncate + the file back to the last complete record, and reuse the torn record's sequence number on its + next append -- never skip past it and leave a gap.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + + path = tmp_path / "events.spool.jsonl" + size_before = path.stat().st_size + with path.open("ab") as stream: + stream.write(b'{"event_id":"e3","sequ') # torn: no closing brace, no trailing newline + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 3 + assert path.stat().st_size == size_before # torn bytes truncated away + + replay = recovered.append({"event_id": "e3-retry"}) + assert replay.sequence == 3 + + ids_in_order = [record.decode()["event_id"] for record in recovered.records()] + assert ids_in_order == ["e1", "e2", "e3-retry"] + + +def test_spool_recovers_from_a_torn_tail_with_no_trailing_newline_at_all(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + # Drop the final newline entirely, simulating a crash after the JSON bytes landed but before + # the trailing separator (and its fsync) did. + data = path.read_bytes() + assert data.endswith(b"\n") + path.write_bytes(data[:-1]) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 1 + assert recovered.records() == [] + assert recovered.append({"event_id": "e1-retry"}).sequence == 1 + + +def test_spool_recovery_degrades_on_mid_file_corruption_instead_of_raising(tmp_path: Path) -> None: + """N8 (supersedes the old B1-era "raises" posture): a bad line with COMPLETE records after it + (a trailing `\\n` of its own) is genuine corruption, never a torn tail -- B1 already forbids + silently renumbering past it, but raising unconditionally was the OTHER extreme: it discarded + even the records BEFORE the corruption, permanently, including a future terminal event. Recovery + must instead succeed, read the readable prefix, and flag the corruption once.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + spool.append({"event_id": "e3"}) + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[1] = b"{not valid json but has a trailing newline" # complete line, unparseable + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) # must NOT raise + assert recovered.is_corrupt + assert recovered.corruption_offset is not None + assert [r.decode()["event_id"] for r in recovered.records()] == ["e1"] + # Never renumbers past the corruption: e1 was sequence 1, so the next allocation is 2 -- even + # though the file still physically contains e2/e3 at sequences 2/3 beyond the corrupt line. + assert recovered.next_sequence == 2 + + +def test_spool_recovery_ships_a_record_appended_after_the_corruption_via_the_seek_path( + tmp_path: Path, +) -> None: + """N8, ranked missing test 5: the readable prefix is still usable AND the stream keeps working + going forward -- a new record appended after recovery (standing in for "the terminal event + still ships") is reachable via `pending_since_watermark()`'s seek path (what `EventsClient.flush` + actually uses), even though a full `records()` scan still stops at the untouched corrupt byte.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + spool.append({"event_id": "e3"}) + spool.advance_watermark(1) # e1 already acked before the corruption is ever discovered + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[1] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + assert recovered.next_sequence == 2 # only e1 (pre-corruption) counted; watermark=1 agrees + + terminal = recovered.append({"event_id": "terminal"}) + assert terminal.sequence == 2 + pending = recovered.pending_since_watermark() + assert [r.decode()["event_id"] for r in pending] == ["terminal"] + + +def test_spool_compact_through_refuses_on_a_corrupt_spool_instead_of_destroying_records( + tmp_path: Path, +) -> None: + # P1, second victim: `_rewrite_retaining` (what BOTH `compact_through` and `drop_many` funnel + # through) used to rewrite the file from ONLY the records-before-corruption prefix, so a rewrite + # on a corrupt spool silently destroyed every intact record past the corrupt byte too -- + # regardless of `compact_through`'s own watermark clamp, which only bounds WHICH sequences it + # asks to keep, not what the underlying rewrite can actually see. Must now refuse entirely + # rather than lose an already-durable, never-sent record this way. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[0] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + + terminal = recovered.append({"event_id": "event_terminal"}) + recovered.advance_watermark(terminal.sequence) # fully acked -- would normally be compactable + recovered.compact_through(terminal.sequence) + + assert b"event_terminal" in path.read_bytes(), ( + "P1 regression: compact_through destroyed a record on a corrupt spool" + ) + + +def test_spool_records_degrades_on_corruption_instead_of_raising( + tmp_path: Path, +) -> None: + # N8: records() shares _iter_complete_records with _recover -- corruption discovered via a + # fresh read (not at construction time) degrades the same way: the readable prefix is still + # returned, never a bare exception (nor the old typed raise) a caller has no reason to catch. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + with path.open("ab") as stream: + stream.write(b"{not valid json}\n") # complete (trailing \n), unparseable + assert not spool.is_corrupt + result = spool.records() + assert [r.decode()["event_id"] for r in result] == ["e1"] + assert spool.is_corrupt + assert spool.corruption_offset is not None + + +def test_spool_corruption_is_logged_once_not_once_per_read(tmp_path: Path, caplog: Any) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + with path.open("ab") as stream: + stream.write(b"{not valid json}\n") + with caplog.at_level(logging.ERROR, logger="fi.alk.harness.outbound"): + spool.records() + spool.records() + spool.records() + corruption_logs = [r for r in caplog.records if "spool corrupt" in r.message] + assert len(corruption_logs) == 1 + + +def test_spool_failed_append_is_a_true_no_op(tmp_path: Path) -> None: + """B1 part 1: a write that raises mid-flush must leave the file exactly as it was before the + attempt (truncated back to size_before, fsync'd) so the retry reuses the same sequence number + on clean ground -- never merges torn bytes with the retry's complete record into one + unparseable line.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "ok1"}) + path = tmp_path / "events.spool.jsonl" + size_before = path.stat().st_size + + class Boom(Exception): + pass + + original_open = Path.open + + def failing_open(self: Path, mode: str = "r", *a: Any, **kw: Any) -> Any: + if self == path and mode == "ab": + raise Boom("simulated write failure") + return original_open(self, mode, *a, **kw) + + Path.open = failing_open # type: ignore[method-assign] + try: + with pytest.raises(Boom): + spool.append({"event_id": "boom"}) + finally: + Path.open = original_open # type: ignore[method-assign] + + assert path.stat().st_size == size_before + + replay = spool.append({"event_id": "ok2"}) + assert replay.sequence == 2 # reused, not skipped + assert [r.decode()["event_id"] for r in spool.records()] == ["ok1", "ok2"] + + +def test_spool_fsyncs_the_directory_once_after_first_file_creation_only(tmp_path: Path) -> None: + # M2: a new file's directory entry isn't durable just because the file's own data is fsync'd. + # The guard flag means exactly ONE extra fsync (the directory's) on the append that creates the + # file, and none on later appends to the same, now-existing file. + calls: list[int] = [] + original_fsync = os.fsync + + def counting_fsync(fd: int) -> None: + calls.append(fd) + original_fsync(fd) + + os.fsync = counting_fsync # type: ignore[assignment] + try: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + before = len(calls) + spool.append({"event_id": "e1"}) + after_first = len(calls) + spool.append({"event_id": "e2"}) + after_second = len(calls) + finally: + os.fsync = original_fsync # type: ignore[assignment] + + assert after_first - before == 2 # the record's own fsync + the one-time directory fsync + assert after_second - after_first == 1 # just the record's own fsync -- no repeat directory sync + + +def test_canonical_bytes_never_contains_a_raw_newline_even_with_embedded_newlines_in_a_string() -> ( + None +): + # The recovery rule's line-per-record framing depends on this holding for every value this + # module ever canonicalizes: json.dumps escapes control characters inside strings, so an + # embedded "\n" in a field value becomes the two-byte escape `\\n`, never a raw newline byte. + # `OutboundSpool.append` asserts this invariant defensively (`outbound_spool_record_unframable`) + # but the assertion is unreachable through this function -- there is no dict this module can be + # asked to canonicalize that trips it. + assert b"\n" not in canonical_bytes({"message": "line one\nline two"}) + + +def test_spool_watermark_starts_at_zero_and_advances(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) + assert spool.watermark() == 0 + assert [r.sequence for r in spool.pending_since_watermark()] == [1, 2, 3, 4, 5] + + spool.advance_watermark(3) + assert spool.watermark() == 3 + assert [r.sequence for r in spool.pending_since_watermark()] == [4, 5] + + +def test_spool_watermark_rejects_a_regression_as_untrusted_input(tmp_path: Path) -> None: + # v1.3/M7: acked_through_sequence is untrusted platform input -- a value below the current + # watermark is rejected (typed error), not silently ignored; the watermark stays unchanged. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(0) + assert excinfo.value.code == "outbound_spool_watermark_out_of_range" + assert spool.watermark() == 1 + + +def test_spool_watermark_rejects_a_value_at_or_above_next_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) # next_sequence becomes 2 + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(2) # not yet allocated + assert excinfo.value.code == "outbound_spool_watermark_out_of_range" + assert spool.watermark() == 0 + + +def test_spool_watermark_advancing_to_the_same_value_is_a_no_op(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + spool.advance_watermark(1) # repeating the same ack must not raise + assert spool.watermark() == 1 + + +def test_spool_watermark_is_durable_across_a_restart(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + first.append({"event_id": "e0"}) + first.advance_watermark(1) + + OutboundSpool._forget_for_tests(tmp_path, "events") + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second.watermark() == 1 + + +def test_spool_watermark_degrades_to_zero_on_a_corrupt_file_instead_of_wedging(tmp_path: Path) -> None: + # M1: a corrupt/torn watermark file must not permanently brick the spool -- re-sending + # already-acked events is safe (at-least-once + dedupe on event_id); raising forever is not. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + watermark_path = tmp_path / "events.spool.watermark.json" + assert watermark_path.exists() + watermark_path.write_text("not json") + assert spool.watermark() == 0 + # And it self-heals: a fresh valid advance still works normally afterward. + spool.advance_watermark(1) + assert spool.watermark() == 1 + + +def test_spool_advance_watermark_leaves_no_leftover_temp_files(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + assert list(tmp_path.glob("*.tmp.*")) == [] + + +def test_advance_watermark_raises_on_a_forked_instance(tmp_path: Path) -> None: + # P7: `advance_watermark` is the one operation that DURABLY destroys delivery state -- once a + # value is written, `pending_since_watermark()` can never return anything at or below it again + # -- yet it was the one mutating method the N25 fork guard skipped. A forked child advancing the + # parent's watermark would silently orphan every record below the new value, the N1 outcome by + # a different route. `_poison_after_fork` is what `os.register_at_fork`'s `after_in_child` hook + # actually calls; invoking it directly simulates the fork without an actual `os.fork()`, which a + # shared test process can't safely drive. + from fi.alk.harness.outbound import _poison_after_fork + + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + _poison_after_fork() + + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(1) + assert excinfo.value.code == "outbound_spool_forked" + + +def test_spool_recovery_seeds_next_sequence_from_the_watermark_when_the_log_is_lost( + tmp_path: Path, +) -> None: + """M5: a watermark ahead of the log (an M2-style lost file, or a compacted/dropped log) must + seed next_sequence from the watermark, not just the empty/short log -- otherwise the allocator + reissues sequence numbers the platform already closed.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(7): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(7) + (tmp_path / "events.spool.jsonl").write_bytes(b"") # simulate the log vanishing + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 8 + assert recovered.append({"event_id": "e7"}).sequence == 8 + + +def test_spool_recovery_seeds_next_sequence_from_the_watermark_when_the_file_is_missing_entirely( + tmp_path: Path, +) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(4): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + (tmp_path / "events.spool.jsonl").unlink() + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 5 + + +def test_spool_drop_is_pure_physical_removal_and_never_touches_the_watermark( + tmp_path: Path, +) -> None: + # N1: an earlier version had `drop` also advance the watermark to `sequence` -- that let an + # UNTRUSTED `rejected[].sequence` silently orphan every pending record below it, bypassing + # `advance_watermark`'s own M7 clamp entirely. `drop`/`drop_many` are now pure physical removal; + # the batch-level `advance_watermark(acked_through_sequence)` is the only chokepoint. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) # sequences 1..5 + spool.drop(3) + remaining = [r.decode()["event_id"] for r in spool.records()] + assert remaining == ["e0", "e1", "e3", "e4"] # e2 was assigned sequence 3 + assert spool.watermark() == 0 # drop() must NEVER move the watermark + + +def test_spool_drop_many_batches_every_removal_into_one_rewrite(tmp_path: Path) -> None: + # N14: a batch of rejections is one _rewrite_retaining call, not one per sequence -- exercised + # indirectly here by checking the end result of a single drop_many call over a set. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(6): + spool.append({"event_id": f"e{i}"}) # sequences 1..6 + spool.drop_many({2, 4, 6}) + remaining = [r.decode()["event_id"] for r in spool.records()] + assert remaining == ["e0", "e2", "e4"] + assert spool.watermark() == 0 # still untouched -- drop_many never advances it either + + +def test_spool_drop_many_with_an_empty_set_is_a_no_op(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.drop_many(set()) + assert [r.decode()["event_id"] for r in spool.records()] == ["e0"] + + +def test_spool_compact_through_drops_acked_records_and_keeps_pending_ones(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(6): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + spool.compact_through(4) + assert [r.decode()["event_id"] for r in spool.records()] == ["e4", "e5"] + + # The allocator survives a restart correctly even though history was compacted away. + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 7 + assert recovered.append({"event_id": "e6"}).sequence == 7 + + +def test_spool_compact_through_clamps_to_the_watermark_never_drops_pending_records( + tmp_path: Path, +) -> None: + # Fail-safe: a caller passing a too-high sequence must not be able to compact away undelivered + # records -- compaction is clamped to whatever the platform has actually acknowledged. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(2) + spool.compact_through(10**9) + assert [r.decode()["event_id"] for r in spool.records()] == ["e2", "e3", "e4"] + + +def test_spool_pending_since_watermark_uses_the_cached_offset_to_seek(tmp_path: Path) -> None: + # P11: previously this only asserted the RESULT ([5..10]), which the generic records_after() + # fallback produces identically -- it could not fail if the seek path regressed to the + # fallback. Monkeypatching records_after to raise pins the claim: pending_since_watermark() + # must succeed WITHOUT ever calling it, proving the cached-offset seek path was actually taken. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(10): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + + def _must_not_be_called(sequence: int) -> Any: + raise AssertionError("pending_since_watermark fell back to records_after instead of seeking") + + spool.records_after = _must_not_be_called # type: ignore[method-assign] + pending = spool.pending_since_watermark() + assert [r.sequence for r in pending] == list(range(5, 11)) + + +def test_unsequenced_spool_assigns_no_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + record = spool.append({"scenario_key": "k1", "status": "passed"}) + assert record.sequence is None + assert record.decode() == {"scenario_key": "k1", "status": "passed"} + + +def test_unsequenced_spool_rejects_a_caller_supplied_sequence_key(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError) as excinfo: + spool.append({"sequence": 1, "scenario_key": "k1"}) + # MIN-9: distinct from the accessor-misuse code below -- "caller passed a sequence key into an + # unsequenced append" and "caller called a sequence-only method" are different faults. + assert excinfo.value.code == "outbound_spool_caller_supplied_sequence" + + +@pytest.mark.parametrize( + "member", ["next_sequence", "watermark", "pending_since_watermark", "records_after", "drop", "compact_through"] +) +def test_unsequenced_spool_rejects_sequence_only_operations(tmp_path: Path, member: str) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError) as excinfo: + if member == "next_sequence": + spool.next_sequence + elif member in ("records_after", "drop", "compact_through"): + getattr(spool, member)(1) + else: + getattr(spool, member)() + assert excinfo.value.code == "outbound_spool_unsequenced" + + +def test_unsequenced_spool_rejects_advance_watermark(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError): + spool.advance_watermark(1) + + +def test_separate_streams_do_not_share_sequence_numbers(tmp_path: Path) -> None: + events = OutboundSpool(tmp_path, "events", sequenced=True) + manifests = OutboundSpool(tmp_path, "artifact_manifest", sequenced=False) + events.append({"event_id": "e1"}) + events.append({"event_id": "e2"}) + manifests.append({"complete": False}) + assert events.next_sequence == 3 + assert [r.sequence for r in events.records()] == [1, 2] + assert [r.sequence for r in manifests.records()] == [None] + + +def test_spool_append_is_safe_under_concurrent_callers(tmp_path: Path) -> None: + # "one allocator, one lock" -- concurrent appends from multiple threads must still produce a + # contiguous, gap-free, duplicate-free sequence. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + sequences: list[int] = [] + lock = threading.Lock() + + def worker(start: int) -> None: + for i in range(25): + record = spool.append({"event_id": f"t{start}-{i}"}) + with lock: + sequences.append(record.sequence) + + threads = [threading.Thread(target=worker, args=(t,)) for t in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert sorted(sequences) == list(range(1, 201)) + assert spool.next_sequence == 201 + + +def test_spool_creates_its_root_directory(tmp_path: Path) -> None: + root = tmp_path / "nested" / "outbound" + assert not root.exists() + OutboundSpool(root, "events", sequenced=True) + assert root.is_dir() + + +def test_spool_root_directory_is_created_with_restrictive_permissions(tmp_path: Path) -> None: + # MIN-10: the spool holds event payloads and receipt bodies in a multi-user sandbox. + root = tmp_path / "outbound" + OutboundSpool(root, "events", sequenced=True) + assert (root.stat().st_mode & 0o777) == 0o700 + + +def test_spool_records_survive_being_read_back_after_many_appends(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(50): + spool.append({"event_id": f"e{i}", "n": i}) + records = spool.records() + assert len(records) == 50 + assert [r.decode()["n"] for r in records] == list(range(50)) + assert (tmp_path / "events.spool.jsonl").is_file() + + +# --- M6: one allocator per (root, name) ----------------------------------------------------- + + +def test_same_stream_name_returns_the_cached_instance_not_a_second_allocator(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second is first + sequences = [first.append({"event_id": "a"}).sequence, second.append({"event_id": "b"}).sequence] + assert sequences == [1, 2] # one allocator regardless of how many handles point at it + + +def test_same_stream_name_with_a_conflicting_sequenced_flag_raises(tmp_path: Path) -> None: + OutboundSpool(tmp_path, "events", sequenced=True) + with pytest.raises(OutboundSpoolError) as excinfo: + OutboundSpool(tmp_path, "events", sequenced=False) + assert excinfo.value.code == "outbound_spool_sequenced_mismatch" + + +def test_flock_rejects_a_second_lock_holder_on_the_same_lock_file(tmp_path: Path) -> None: + # M6: cross-PROCESS protection -- the in-process registry above only protects against a second + # Python-level instance in this same interpreter; a genuinely separate process trying to open + # the same lock file must be rejected by the OS-level flock regardless of the registry. + OutboundSpool(tmp_path, "events", sequenced=True) # holds the flock via its own fd + lock_path = tmp_path / "events.spool.lock" + fd = os.open(str(lock_path), os.O_RDWR) + try: + with pytest.raises(OSError): + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(fd) + + +def test_a_forgotten_spool_can_be_reacquired_after_the_lock_is_released(tmp_path: Path) -> None: + OutboundSpool(tmp_path, "events", sequenced=True) + OutboundSpool._forget_for_tests(tmp_path, "events") # releases the flock fd too + # A fresh construction must succeed cleanly -- proves _forget_for_tests actually released the + # OS-level lock, not just the in-process cache entry. + fresh = OutboundSpool(tmp_path, "events", sequenced=True) + assert fresh.next_sequence == 1 + + +def test_close_releases_the_lock_and_evicts_the_registry(tmp_path: Path) -> None: + # N26: OutboundSpool had no supported teardown -- close() is the real one (_forget_for_tests is + # test-only and reaches for it internally now). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.close() + fresh = OutboundSpool(tmp_path, "events", sequenced=True) # must not raise outbound_spool_locked + assert fresh is not spool + assert fresh.next_sequence == 1 + spool.close() # idempotent -- a second close on the old instance must not raise + + +def test_close_poisons_the_instance_so_a_closed_spool_cannot_append_with_no_lock_held( + tmp_path: Path, +) -> None: + # P2: close() evicted the registry entry and released the flock fd, but left the closed + # instance itself fully mutable -- `_initialized` stayed True and nothing else guarded it, so a + # caller still holding the closed reference could keep appending with NO lock held, and a fresh + # construction for the same key would allocate an independent, unaware second `_next_sequence`. + # `close()` must poison the instance too, not just the registry slot. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.close() + + with pytest.raises(OutboundSpoolError) as excinfo: + spool.append({"event_id": "after-close"}) + assert excinfo.value.code == "outbound_spool_closed" + + with pytest.raises(OutboundSpoolError): + spool.advance_watermark(1) + + # A fresh instance for the same key is the ONLY live allocator now -- it continues the SAME + # sequence the closed instance left off at, rather than starting a second, independent one. + fresh = OutboundSpool(tmp_path, "events", sequenced=True) + fresh_record = fresh.append({"event_id": "fresh"}) + assert fresh_record.sequence == 2 + assert [r.decode()["event_id"] for r in fresh.records()] == ["e1", "fresh"] + + +def test_failed_construction_does_not_poison_the_registry_for_the_next_attempt(tmp_path: Path) -> None: + # N4, ranked missing test 4: a construction that fails PARTWAY (after acquiring the process + # lock, before finishing) must not leave a half-built instance registered -- the next + # construction attempt for the SAME key gets a typed error reflecting its OWN failure, never + # AttributeError (a half-built instance missing _sequenced) or a spurious outbound_spool_locked + # (a leaked fd from the first attempt). + root = tmp_path / "n4" + lock_path = root / "events.spool.lock" + root.mkdir() + # Simulate "another process" already holding the flock -- this is exactly what + # _acquire_process_lock hits AFTER self._sequenced is already set, so it reproduces the review's + # second failure mode (a failure after the half-built-instance point, not before it). + held_fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) + fcntl.flock(held_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + with pytest.raises(OutboundSpoolError) as excinfo: + OutboundSpool(root, "events", sequenced=True) + assert excinfo.value.code == "outbound_spool_locked" + finally: + os.close(held_fd) # release the "other process"'s hold + + # The registry must have NOTHING registered for this key -- construction now succeeds cleanly, + # not with AttributeError ('_sequenced') and not with a second outbound_spool_locked from a + # leaked fd. + spool = OutboundSpool(root, "events", sequenced=True) + assert spool.next_sequence == 1 + + +# ================================================================================================= +# PART 2 -- Transport: error map, retry/backoff, typed receipt/manifest models, channel clients. +# ================================================================================================= + + +# --- classify_response (the closed error map) --------------------------------------------------- + + +def test_classify_response_returns_none_for_every_2xx() -> None: + assert classify_response(200, {}, attempt=1) is None + assert classify_response(201, {"status": "stored"}, attempt=1) is None + assert classify_response(299, {}, attempt=1) is None + + +def test_classify_response_network_failure_is_retryable_connectivity() -> None: + error = classify_response(None, None, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.domain is FailureDomain.CONNECTIVITY + assert error.code == "network_error" + + +@pytest.mark.parametrize("status", [401, 403]) +def test_classify_response_401_403_are_fenced_never_retried(status: int) -> None: + error = classify_response(status, {"error": "e", "message": "m", "retryable": False}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.FENCED + assert error.domain is None # "never an infra retry" -- not a FailureDomain-tagged outcome + + +def test_classify_response_404_retries_twice_then_channel_failed_on_the_third() -> None: + first = classify_response(404, {"error": "not_found", "message": "m"}, attempt=1) + second = classify_response(404, {"error": "not_found", "message": "m"}, attempt=2) + third = classify_response(404, {"error": "not_found", "message": "m"}, attempt=3) + assert first is not None and first.outcome is ChannelOutcome.RETRYABLE + assert second is not None and second.outcome is ChannelOutcome.RETRYABLE + assert third is not None + assert third.outcome is ChannelOutcome.CHANNEL_FAILED + assert third.domain is FailureDomain.PLATFORM_SYNC + + +def test_classify_response_413_is_budget_exceeded() -> None: + error = classify_response(413, {"error": "artifact_budget_exceeded", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert error.status_code == 413 + + +def test_classify_response_413_without_artifact_budget_exceeded_is_permanent_item() -> None: + # N7: 413 is Channel 3's code specifically -- a 413 that does NOT report + # artifact_budget_exceeded (e.g. an events batch too big for the platform's own ingress cap) + # must not be mislabeled BUDGET_EXCEEDED on a channel with no such concept. + error = classify_response(413, {"error": "request_entity_too_large", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + assert error.status_code == 413 + + error_no_body = classify_response(413, None, attempt=1) + assert error_no_body is not None + assert error_no_body.outcome is ChannelOutcome.PERMANENT_ITEM + + +def test_classify_response_404_domain_is_platform_sync_on_every_attempt() -> None: + # N29: a 404 is never a connectivity fault under §4.6 -- PLATFORM_SYNC on attempts 1-2 too, not + # just the third (cosmetic before; the inconsistency invited a wrong read). + first = classify_response(404, {"error": "not_found", "message": "m"}, attempt=1) + second = classify_response(404, {"error": "not_found", "message": "m"}, attempt=2) + assert first is not None and first.domain is FailureDomain.PLATFORM_SYNC + assert second is not None and second.domain is FailureDomain.PLATFORM_SYNC + + +def test_classify_response_unlisted_3xx_is_permanent_not_retryable() -> None: + # N27: the contract never contemplates a 3xx (every endpoint ends in "/" precisely so Django's + # POST-redirect problem never arises) -- a 301 from a mis-slashed endpoint should surface as a + # permanent misconfiguration, not loop max_attempts times before giving up as connectivity. + error = classify_response(301, {"error": "e", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + + +def test_classify_response_429_is_retryable_and_carries_retry_after() -> None: + error = classify_response( + 429, {"error": "rate_limited", "message": "m"}, attempt=1, retry_after_seconds=7.0 + ) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.retry_after_seconds == 7.0 + + +@pytest.mark.parametrize("status", [400, 409, 422]) +def test_classify_response_400_409_422_are_permanent_item(status: int) -> None: + error = classify_response(status, {"error": f"e{status}", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + assert error.domain is None + + +def test_classify_response_unlisted_4xx_is_the_permanent_catch_all() -> None: + error = classify_response(418, {"error": "teapot", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + + +@pytest.mark.parametrize("status", [500, 502, 503]) +def test_classify_response_5xx_is_retryable_connectivity(status: int) -> None: + error = classify_response(status, {"error": "e", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.domain is FailureDomain.CONNECTIVITY + + +# --- compute_backoff_seconds (full jitter) -------------------------------------------------------- + + +def test_compute_backoff_seconds_full_jitter_formula() -> None: + # rng pinned to 1.0 -> exercises the pure ceiling formula, unjittered. + assert compute_backoff_seconds(1, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 1.0 + assert compute_backoff_seconds(2, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 2.0 + assert compute_backoff_seconds(3, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 4.0 + + +def test_compute_backoff_seconds_caps_at_max_backoff() -> None: + assert compute_backoff_seconds(10, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 15.0 + + +def test_compute_backoff_seconds_scales_by_rng() -> None: + assert compute_backoff_seconds(3, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 0.5) == 2.0 + + +# --- format_rfc3339_millis ------------------------------------------------------------------------- + + +def test_format_rfc3339_millis_matches_the_contracts_wire_form() -> None: + assert format_rfc3339_millis(NOW) == "2026-08-25T10:14:03.412Z" + assert is_valid_rfc3339_millis(format_rfc3339_millis(NOW)) + + +def test_format_rfc3339_millis_rejects_naive_datetimes() -> None: + with pytest.raises(ValueError): + format_rfc3339_millis(datetime(2026, 8, 25)) + + +def test_is_valid_rfc3339_millis_rejects_the_default_pydantic_offset_form() -> None: + # The exact failure mode `format_rfc3339_millis` exists to prevent: pydantic's default + # datetime JSON serialization uses a `+00:00` offset and six-digit microseconds, not `Z` and + # milliseconds -- this string is what NOT using the helper would have produced. + assert not is_valid_rfc3339_millis("2026-08-25T10:14:03.412000+00:00") + + +# --- ArtifactBudgetTracker ------------------------------------------------------------------------- + + +def test_artifact_budget_tracker_reserved_kinds_always_admitted() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=10) + assert tracker.would_admit(ArtifactKind.RESULT, 1_000_000, digest="d1") + tracker.record(ArtifactKind.RESULT, 1_000_000, digest="d1") + # Reserved kinds are counted (for accounting) but never refused, even once over budget. + assert tracker.admitted_bytes == 1_000_000 + assert tracker.would_admit(ArtifactKind.BUILD, 999, digest="d2") + + +def test_artifact_budget_tracker_refuses_non_reserved_once_budget_is_exhausted() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=100) + assert tracker.would_admit(ArtifactKind.TRACE, 60, digest="d1") + tracker.record(ArtifactKind.TRACE, 60, digest="d1") + assert tracker.would_admit(ArtifactKind.TRACE, 40, digest="d2") + tracker.record(ArtifactKind.TRACE, 40, digest="d2") + assert not tracker.would_admit(ArtifactKind.LOG, 1, digest="d3") + + +def test_artifact_budget_tracker_duplicate_digest_is_free() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=10) + tracker.record(ArtifactKind.LOG, 10, digest="dup") + assert tracker.admitted_bytes == 10 + assert tracker.would_admit(ArtifactKind.LOG, 10, digest="dup") + tracker.record(ArtifactKind.LOG, 10, digest="dup") + assert tracker.admitted_bytes == 10 # unchanged -- already counted + + +def test_priority_class_orders_reserved_recordings_other() -> None: + # N16: reserved (0) > recordings (1) > trace/log/other (2), per the contract's three-tier + # budget partition. + assert priority_class(ArtifactKind.RESULT) == 0 + assert priority_class(ArtifactKind.BUILD) == 0 + assert priority_class(ArtifactKind.TRANSCRIPT) == 0 + assert priority_class(ArtifactKind.TOOL_TRACE) == 0 + assert priority_class(ArtifactKind.RECORDING_COMBINED) == 1 + assert priority_class(ArtifactKind.RECORDING_STEREO) == 1 + assert priority_class(ArtifactKind.TRACE) == 2 + assert priority_class(ArtifactKind.LOG) == 2 + assert priority_class(ArtifactKind.OTHER) == 2 + + +def test_artifact_budget_tracker_reserves_recording_headroom_from_trace_log_other() -> None: + # N16: a non-zero recording_headroom_bytes shrinks what a trace/log/other candidate may + # consume, leaving room for recordings not yet seen -- default (0) behavior is unaffected + # (covered by the pre-existing tracker tests above). + tracker = ArtifactBudgetTracker(max_artifact_bytes=100, recording_headroom_bytes=30) + # 70 bytes remain after the 30-byte reservation -- a 71-byte trace is refused, a 70-byte one is not. + assert not tracker.would_admit(ArtifactKind.TRACE, 71, digest="d1") + assert tracker.would_admit(ArtifactKind.TRACE, 70, digest="d2") + # Recordings are NOT subject to the headroom reservation themselves -- only class-2 candidates. + assert tracker.would_admit(ArtifactKind.RECORDING_COMBINED, 100, digest="d3") + + +# --- ResultReceiptDraft / build_result_receipt / build_skipped_receipt ----------------------------- + + +def _passed_receipt(**overrides: Any) -> dict[str, Any]: + base = dict( + job_id="j1", + attempt_id="a1", + attempt_number=1, + scenario_key="suspended-account-blocked", + scenario_id="sid-1", + scenario_attempt=1, + world_index=2, + status=ScenarioStatus.PASSED, + sub_goals=[{"name": "n", "held": True, "reason": None, "judged": False}], + evaluations=[ + {"name": "customer_agent_task_completion", "kind": "metric", "score": 0.86, "reason": "r"}, + {"name": "checked_refund", "kind": "checkpoint", "passed": True, "reason": "r"}, + ], + call={ + "started_at": format_rfc3339_millis(NOW), + "ended_at": format_rfc3339_millis(LATER), + "duration_ms": 184211, + "turns": 5, + "transcript_artifact": "sha256:" + "a" * 64, + "recording_artifacts": ["sha256:" + "b" * 64], + }, + failure=None, + ) + base.update(overrides) + return build_result_receipt(**base) + + +def test_build_result_receipt_matches_the_contract_shaped_example() -> None: + receipt = _passed_receipt() + assert is_valid_digest(receipt["digest"]) + assert receipt["status"] == "passed" + assert receipt["digest"] == whole_object_digest({k: v for k, v in receipt.items() if k != "digest"}) + ResultReceiptDraft.model_validate(receipt) # round-trips through the model unchanged + + +def test_build_skipped_receipt_has_the_exact_contract_shape() -> None: + receipt = build_skipped_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k2", scenario_id="sid-2" + ) + assert receipt["scenario_attempt"] == 1 + assert receipt["world_index"] is None + assert receipt["sub_goals"] == [] + assert receipt["evaluations"] == [] + assert receipt["call"] is None + assert receipt["failure"] is None + assert receipt["status"] == "skipped" + + +def test_skipped_shape_is_enforced_even_outside_the_synthesizing_helper() -> None: + with pytest.raises(ValidationError): + # world_index must be null for a skipped receipt -- even via the general-purpose builder. + _passed_receipt(status=ScenarioStatus.SKIPPED, world_index=0, sub_goals=[], evaluations=[], call=None) + + +def test_errored_receipt_requires_a_failure() -> None: + with pytest.raises(ValidationError): + _passed_receipt(status=ScenarioStatus.ERRORED, call=None, failure=None) + + errored = _passed_receipt( + status=ScenarioStatus.ERRORED, + call=None, + failure={"domain": "agent", "stage": "running", "code": "agent_crashed", "message": "m"}, + ) + assert is_valid_digest(errored["digest"]) + + +def test_receipt_evaluations_discriminated_union_rejects_a_mixed_up_kind() -> None: + with pytest.raises(ValidationError): + _passed_receipt(evaluations=[{"name": "n", "kind": "metric", "passed": True, "reason": "r"}]) + + +def test_receipt_call_timestamp_must_be_the_canonical_rfc3339_millis_form() -> None: + with pytest.raises(ValidationError): + _passed_receipt(call={**_passed_receipt()["call"], "started_at": "2026-08-25T10:14:03.412000+00:00"}) + + +def test_a_tampered_receipt_digest_is_rejected() -> None: + receipt = _passed_receipt() + receipt["digest"] = "sha256:" + "0" * 64 + with pytest.raises(ValidationError): + ResultReceiptDraft.model_validate(receipt) + + +def test_receipt_digest_mismatch_names_the_unset_default_field() -> None: + # N23: a caller whose raw `call` dict omits `recording_artifacts` (an optional field with a + # default) computes a digest over an object without it; the model's own re-derivation fills in + # `recording_artifacts: []`, so the two digests disagree. The error should name the differing + # field -- a bare `receipt_digest_mismatch` was the review's named "opaque, undiagnosable" gap. + call_without_recording_artifacts = { + "started_at": format_rfc3339_millis(NOW), + "ended_at": format_rfc3339_millis(LATER), + "duration_ms": 100, + "turns": 1, + "transcript_artifact": None, + # "recording_artifacts" intentionally omitted -- CallSummary fills it via default_factory. + } + with pytest.raises(ValidationError) as excinfo: + build_result_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid", + scenario_attempt=1, world_index=0, status=ScenarioStatus.PASSED, + sub_goals=[{"name": "n", "held": True, "reason": None, "judged": False}], + evaluations=[], + call=call_without_recording_artifacts, + failure=None, + ) + assert "call.recording_artifacts" in str(excinfo.value) + + +# --- ArtifactManifestDraft / build_artifact_manifest ------------------------------------------------ + + +def test_build_artifact_manifest_matches_the_contract_shaped_example() -> None: + manifest = build_artifact_manifest( + job_id="j1", + attempt_id="a1", + attempt_number=1, + entries=[{"artifact_id": "sha256:" + "1" * 64, "kind": "result", "size": 1048576, "scenario_key": "k"}], + complete=True, + ) + assert is_valid_digest(manifest["digest"]) + ArtifactManifestDraft.model_validate(manifest) + + +def test_manifest_complete_flag_changes_the_digest() -> None: + entries = [{"artifact_id": "sha256:" + "2" * 64, "kind": "log", "size": 10, "scenario_key": None}] + complete_manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=entries, complete=True) + partial_manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=entries, complete=False) + # "a later `complete: true` manifest is a distinct document that supersedes an earlier + # `complete: false` one from the same attempt." + assert complete_manifest["digest"] != partial_manifest["digest"] + + +def test_manifest_entry_rejects_a_malformed_artifact_id() -> None: + with pytest.raises(ValidationError): + build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": "not-a-digest", "kind": "log", "size": 1, "scenario_key": None}], + complete=True, + ) + + +# --- FakePlatform: in-process, no-sockets Transport ------------------------------------------------ + + +class FakePlatform: + """A `Transport` implementation backed by real, persistent server-side state -- no sockets, no + `requests` -- so the channel clients above are exercised through the exact same interface + `RequestsTransport` implements. `programmed` lets a test inject canned failures (an `Exception` + instance to raise, or a `TransportResponse` to return) that are consumed FIFO before requests + fall through to the real per-channel handlers below, which maintain idempotent state matching + the contract's own dedupe rules (`event_id`, `(job_id, scenario_key)`, artifact digest, + `(attempt_id, digest)`). `crash_after_next_write` lets a test simulate "the platform durably + processed the write, but the response never reached the guest" -- the handler still records + state and THEN raises `TransportError`, so the next request (whether inside the same client + call's retry loop, or from a brand new client after a simulated restart) hits the idempotent + path rather than reprocessing. + """ + + def __init__( + self, + *, + events_url: str, + results_url: str, + artifacts_url: str, + expected_token: str = "tok", + expected_fence: str = "fence1", + ) -> None: + self.events_url = events_url + self.results_url = results_url + self.artifacts_url = artifacts_url + self.manifest_url = artifacts_url + "manifest/" + self.expected_token = expected_token + self.expected_fence = expected_fence + + self._events_by_id: dict[str, dict[str, Any]] = {} + self._watermark = 0 + self._receipts: dict[tuple[str, str], dict[str, Any]] = {} + self._artifacts: dict[str, bytes] = {} + self._manifests: dict[tuple[str, str], dict[str, Any]] = {} + + self.events_to_reject: set[str] = set() + self.budget_remaining: int | None = None + self.reserved_kinds = {"build", "transcript", "tool_trace", "result"} + + self.programmed: list[Exception | TransportResponse] = [] + self.crash_after_next_write = False + self.calls: list[tuple[str, str]] = [] + self.received_headers: list[dict[str, str]] = [] + + def queue(self, item: Exception | TransportResponse) -> None: + self.programmed.append(item) + + def _maybe_crash(self, response: TransportResponse) -> TransportResponse: + if self.crash_after_next_write: + self.crash_after_next_write = False + raise TransportError("simulated crash: response lost after platform-side processing") + return response + + def _check_auth(self, headers: dict[str, str]) -> TransportResponse | None: + # Strengthened per the union review: every real (non-programmed) request must carry the + # bearer + fence -- exercising the auth-header wiring the contract mandates on every one of + # the four endpoints, not just implicitly via "the call succeeded." + if headers.get("Authorization") != f"Bearer {self.expected_token}": + return TransportResponse(401, {"error": "token_invalid", "message": "missing/wrong bearer", "retryable": False}, {}) + if headers.get("X-Harness-Fence") != self.expected_fence: + return TransportResponse(403, {"error": "fence_mismatch", "message": "missing/wrong fence", "retryable": False}, {}) + return None + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: Any = None, + timeout: float = 30.0, + ) -> TransportResponse: + self.calls.append((method, url)) + self.received_headers.append(dict(headers)) + if self.programmed: + item = self.programmed.pop(0) + if isinstance(item, Exception): + raise item + return item + + auth_error = self._check_auth(headers) + if auth_error is not None: + return auth_error + + if method == "POST" and url == self.events_url: + # EventsClient.flush() sends the batch as spooled bytes verbatim via `data=` (N19), not + # `json_body=` -- decode it the same way the real platform would. + if json_body is not None: + parsed_body = json_body + elif isinstance(data, (bytes, bytearray)): + parsed_body = json.loads(data.decode("utf-8")) + else: + parsed_body = {} + return self._handle_events(parsed_body) + if method == "POST" and url == self.results_url: + return self._handle_receipt(json_body or {}) + if method == "POST" and url == self.manifest_url: + return self._handle_manifest(json_body or {}) + if method == "PUT" and url.startswith(self.artifacts_url): + return self._handle_upload(url, headers, data) + raise AssertionError(f"unrouted fake-platform request: {method} {url}") + + def _handle_events(self, body: dict[str, Any]) -> TransportResponse: + assert body.get("schema_version") == "futureagi.harness-event.v1", ( + f"batch envelope missing/wrong schema_version: {body.get('schema_version')!r}" + ) + rejected: list[dict[str, Any]] = [] + max_seq = self._watermark + for event in body["events"]: + sequence = event["sequence"] + max_seq = max(max_seq, sequence) + if event["event_id"] in self.events_to_reject: + rejected.append( + { + "event_id": event["event_id"], + "sequence": sequence, + "code": "deterministic_rejection", + "message": "rejected by fake", + } + ) + continue + self._events_by_id[event["event_id"]] = event + self._watermark = max(self._watermark, max_seq) + return self._maybe_crash( + TransportResponse(200, {"acked_through_sequence": self._watermark, "rejected": rejected}, {}) + ) + + def _handle_receipt(self, body: dict[str, Any]) -> TransportResponse: + # Strengthened per the union review: the real platform verifies the receipt digest -- + # recompute it the same way `whole_object_digest` does and reject a mismatch, so a test that + # tampers with a receipt body actually exercises this path instead of it being silently + # unenforced. + core = {k: v for k, v in body.items() if k != "digest"} + if body.get("digest") != whole_object_digest(core): + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + key = (body["job_id"], body["scenario_key"]) + existing = self._receipts.get(key) + if existing is not None: + if existing["digest"] == body["digest"]: + return self._maybe_crash(TransportResponse(200, {"status": "duplicate"}, {})) + if existing["attempt_number"] >= body["attempt_number"]: + code = ( + "attempt_superseded" + if existing["attempt_number"] > body["attempt_number"] + else "receipt_conflict" + ) + return self._maybe_crash( + TransportResponse(409, {"error": code, "message": "m", "retryable": False}, {}) + ) + self._receipts[key] = body + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + def _handle_upload(self, url: str, headers: dict[str, str], data: Any) -> TransportResponse: + digest_hex = url[len(self.artifacts_url) :].rstrip("/") + body_bytes = data if isinstance(data, (bytes, bytearray)) else b"".join(data) + if digest_hex in self._artifacts: + return self._maybe_crash(TransportResponse(200, {"status": "already_exists"}, {})) + if hashlib.sha256(body_bytes).hexdigest() != digest_hex: + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + kind = headers.get("X-Artifact-Kind") + if ( + self.budget_remaining is not None + and kind not in self.reserved_kinds + and len(body_bytes) > self.budget_remaining + ): + return self._maybe_crash( + TransportResponse( + 413, {"error": "artifact_budget_exceeded", "message": "m", "retryable": False}, {} + ) + ) + self._artifacts[digest_hex] = body_bytes + if self.budget_remaining is not None: + self.budget_remaining -= len(body_bytes) + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + def _handle_manifest(self, body: dict[str, Any]) -> TransportResponse: + # Strengthened per the union review: recompute and verify the manifest digest too. + core = {k: v for k, v in body.items() if k != "digest"} + if body.get("digest") != whole_object_digest(core): + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + key = (body["attempt_id"], body["digest"]) + if key in self._manifests: + return self._maybe_crash(TransportResponse(200, {"status": "duplicate"}, {})) + for entry in body["entries"]: + digest_hex = entry["artifact_id"].split(":", 1)[1] + if digest_hex not in self._artifacts: + return self._maybe_crash( + TransportResponse(422, {"error": "artifact_unknown", "message": entry["artifact_id"], "retryable": False}, {}) + ) + self._manifests[key] = body + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + +ENDPOINTS = { + "events": "https://platform.example/events/", + "results": "https://platform.example/results/", + "artifacts": "https://platform.example/artifacts/", + "scenarios": "https://platform.example/scenarios/", +} + + +def _capabilities() -> HostedCapabilities: + return HostedCapabilities.model_validate( + { + "schema_version": CAPABILITIES_SCHEMA_VERSION, + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "fence": "fence1", + "expires_at": "2026-08-25T12:00:00.000Z", + "token": "tok", + "endpoints": ENDPOINTS, + } + ) + + +def _platform() -> FakePlatform: + return FakePlatform( + events_url=ENDPOINTS["events"], results_url=ENDPOINTS["results"], artifacts_url=ENDPOINTS["artifacts"] + ) + + +def _spooled_log_event(spool: OutboundSpool, event_id: str) -> None: + record = build_event_record( + event_id=event_id, + job_id="j1", + attempt_id="a1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": event_id}, + ) + spool.append(record) + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts_upload", "artifacts_manifest"]) +def test_every_channel_client_sends_authorization_and_fence_headers(tmp_path: Path, channel: str) -> None: + # Ranked missing test 8: one parametrized test over all four endpoints. `FakePlatform` now + # enforces these headers itself (`_check_auth`) -- a wrong/missing one would 401/403 here, so a + # passing run proves the header is both sent AND correct, not merely present in a recorded list. + platform = _platform() + capabilities = _capabilities() + if channel == "events": + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + result = EventsClient(capabilities, spool, platform, sleep=lambda s: None).flush() + assert result.error is None + elif channel == "results": + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + result = ResultsClient(capabilities, platform, sleep=lambda s: None).push(receipt) + assert result.delivered + elif channel == "artifacts_upload": + data = b"auth header check" + digest_hex = hashlib.sha256(data).hexdigest() + result = ArtifactsClient(capabilities, platform, sleep=lambda s: None).upload(digest_hex, data, kind=ArtifactKind.LOG) + assert result.delivered + else: + manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=[], complete=True) + result = ArtifactsClient(capabilities, platform, sleep=lambda s: None).push_manifest(manifest) + assert result.delivered + + assert platform.received_headers # at least one request actually happened + for headers in platform.received_headers: + assert headers.get("Authorization") == "Bearer tok" + assert headers.get("X-Harness-Fence") == "fence1" + + +# --- EventsClient ------------------------------------------------------------------------------ + + +def test_events_client_flush_delivers_and_advances_the_watermark(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(3): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + assert spool.watermark() == 0 + result = client.flush() + assert result.error is None + assert result.delivered_count == 3 + assert result.acked_through_sequence == 3 + assert result.rejected == [] + assert spool.watermark() == 3 + + +def test_events_client_flush_is_a_no_op_transport_call_when_nothing_is_pending(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + result = client.flush() + assert result.delivered_count == 0 + assert platform.calls == [] + + +def test_events_client_never_advances_watermark_on_a_retryable_failure(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportError("boom")) + client = EventsClient( + _capabilities(), spool, platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=1) + ) + result = client.flush() + assert result.error is not None + assert result.error.outcome is ChannelOutcome.RETRYABLE + assert spool.watermark() == 0 # "advancing the spool watermark only on confirmed delivery" + + +def test_events_client_transient_then_success_network_error_then_5xx(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportError("connection refused")) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert result.delivered_count == 1 + assert sleeps == [1.0, 2.0] # base backoff, then doubled -- full jitter pinned to 1.0 + assert len(platform.calls) == 3 + + +def test_events_client_429_honors_retry_after_over_computed_backoff(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "9"})) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [9.0] + + +def test_events_client_retry_after_86400_is_clamped_to_max_backoff(tmp_path: Path) -> None: + # N5, ranked missing test 6: an absurd Retry-After must not be honored verbatim -- clamp to + # retry_policy.max_backoff_seconds (15.0 by default). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue( + TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "86400"}) + ) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [15.0] + + +def test_events_client_retry_after_negative_is_treated_as_absent(tmp_path: Path) -> None: + # N5: a negative Retry-After must not reach `time.sleep` (ValueError) -- treated as absent, the + # caller falls back to the computed full-jitter backoff instead. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue( + TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "-5"}) + ) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [1.0] # the computed backoff for attempt 1, not -5.0 and not a crash + + +def test_perform_with_retry_deadline_refuses_a_new_attempt_once_elapsed() -> None: + from fi.alk.harness.outbound import RetryPolicy, _perform_with_retry + + calls: list[int] = [] + + def perform(attempt: int) -> TransportResponse: + calls.append(attempt) + return TransportResponse(503, {"error": "e", "message": "m", "retryable": True}, {}) + + response, error = _perform_with_retry( + perform, + retry_policy=RetryPolicy(max_attempts=8), + sleep=lambda s: None, + now=lambda: 100.0, + deadline=99.0, # already in the past relative to `now` + ) + assert calls == [] # never even attempted once + assert response is None + assert error is not None + assert error.code == "deadline_exceeded" + + +def test_perform_with_retry_deadline_clamps_the_sleep_to_the_remaining_budget() -> None: + from fi.alk.harness.outbound import RetryPolicy, _perform_with_retry + + clock = [0.0] + + def now() -> float: + return clock[0] + + def perform(_attempt: int) -> TransportResponse: + return TransportResponse(503, {"error": "e", "message": "m", "retryable": True}, {}) + + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock[0] += seconds + + _perform_with_retry( + perform, + retry_policy=RetryPolicy(max_attempts=8, initial_backoff_seconds=100.0, max_backoff_seconds=100.0), + sleep=sleep, + rng=lambda: 1.0, + now=now, + deadline=5.0, # far less than the ~100s computed backoff + ) + assert sleeps[0] == 5.0 # clamped to what remained, not the full computed backoff + + +def test_events_client_deterministic_rejection_advances_watermark_but_never_retries(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_ok") + _spooled_log_event(spool, "event_bad") + platform = _platform() + platform.events_to_reject = {"event_bad"} + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == 2 + assert [item["event_id"] for item in result.rejected] == ["event_bad"] + assert result.delivered_count == 1 + assert spool.watermark() == 2 # "the watermark is highest-processed -- accepted AND rejected" + assert len(platform.calls) == 1 # never retried + + +def test_events_client_physically_drops_a_rejected_record_from_the_spool(tmp_path: Path) -> None: + # P8 alignment: rejected-event handling must go through the spool's own drop(sequence), not + # leave the rejected record sitting in the log forever. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_ok") + _spooled_log_event(spool, "event_bad") + platform = _platform() + platform.events_to_reject = {"event_bad"} + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + client.flush() + remaining_ids = [record.decode()["event_id"] for record in spool.records()] + assert "event_bad" not in remaining_ids + assert "event_ok" in remaining_ids + + +def test_events_client_flush_rejection_does_not_destroy_an_unacked_record_past_a_corrupt_spool( + tmp_path: Path, +) -> None: + # P1 (BLOCKER->MAJOR), reproduces the round-3 review's exact end-to-end scenario through the + # public API: `_rewrite_retaining` (what the rejection-driven `drop_many` funnels through) used + # to rewrite the log from ONLY the records-before-corruption prefix -- so an ordinary, + # contract-defined rejection during `flush()` silently destroyed every intact record past the + # corrupt byte too, including one the platform had NOT even acknowledged yet (the terminal + # event, in the worst case). This is the N1 failure class (silent, zero-diagnostic loss of the + # outbound event stream) reintroduced by the N8/N14 rework's interaction, and it is the one + # thing round 3 was explicitly asked to break. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_1") + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[0] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + + _spooled_log_event(recovered, "event_bad") # sequence 1, appended post-recovery + _spooled_log_event(recovered, "event_terminal") # sequence 2, genuinely never yet acked + + platform = _platform() + # Ack ONLY sequence 1 (rejected) -- sequence 2 (event_terminal) is deliberately left un-acked, + # so it must still be reported pending after this flush, not silently vanish from disk. + platform.queue( + TransportResponse( + 200, + { + "acked_through_sequence": 1, + "rejected": [{"sequence": 1, "code": "event_type_unknown", "message": "m"}], + }, + {}, + ) + ) + client = EventsClient(_capabilities(), recovered, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + + pending_ids = [r.decode()["event_id"] for r in recovered.pending_since_watermark()] + assert pending_ids == ["event_terminal"], ( + "P1 regression: an un-acked record past a corrupt byte was destroyed by the rewrite the " + "rejection triggered" + ) + assert b"event_terminal" in path.read_bytes() + + +def test_events_client_rejects_an_untrusted_acked_through_sequence_and_leaves_watermark_unchanged( + tmp_path: Path, +) -> None: + # M7: acked_through_sequence is untrusted platform input -- a value outside the spool's trusted + # range must not be applied; the flush still reports success (the HTTP call itself delivered). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 10**9, "rejected": []}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.ack_out_of_range is True + assert result.error is None + assert spool.watermark() == 0 + + +def test_events_client_rejected_sequence_outside_the_batch_is_ignored_not_orphaning( + tmp_path: Path, +) -> None: + # N1 (BLOCKER), ranked missing test 1: a `rejected[].sequence` the guest never sent in this + # batch must be ignored -- not dropped, not used to move anything -- so it cannot orphan pending + # records. Reproduces the union review's exact scenario: 10 spooled events, watermark forced to + # 0, a rejected entry naming a sequence far outside [1, 10]. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(10): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + platform.queue( + TransportResponse(200, {"acked_through_sequence": 0, "rejected": [{"event_id": "x9", "sequence": 10**9, "code": "c", "message": "m"}]}, {}) + ) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.rejected == [] # the bogus entry never makes it into the reported rejections + assert spool.watermark() == 0 + assert len(spool.pending_since_watermark()) == 10 # nothing orphaned + assert len(spool.records()) == 10 # nothing physically dropped either + + +def test_events_client_ack_body_missing_acked_through_sequence_sets_ack_missing( + tmp_path: Path, +) -> None: + # N2/N3, ranked missing test 1: a 2xx with no acked_through_sequence at all must not silently + # re-send the same batch forever -- ack_missing is set, watermark stays put, no exception. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"rejected": []}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.ack_missing is True + assert spool.watermark() == 0 + + +@pytest.mark.parametrize( + "body", + [ + {"acked_through_sequence": None, "rejected": []}, + {"acked_through_sequence": "abc", "rejected": []}, + {"acked_through_sequence": True, "rejected": []}, # bool must not pass an int check + {}, + ], +) +def test_events_client_hostile_ack_bodies_never_raise(tmp_path: Path, body: dict[str, Any]) -> None: + # N2, ranked missing test 1: a non-int/missing acked_through_sequence must produce a typed, + # non-exception result, never crash the flusher (`TypeError`/`ValueError` from a bare `int()`). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, body, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.ack_missing is True + assert spool.watermark() == 0 + + +def test_events_client_rejected_is_not_a_list_never_raises(tmp_path: Path) -> None: + # N2: `rejected: null` (or any non-list) must be treated as empty, not crash `sorted()`/iteration. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 1, "rejected": None}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.rejected == [] + assert spool.watermark() == 1 + + +def test_events_client_rejected_list_of_non_dicts_never_raises(tmp_path: Path) -> None: + # N2: `rejected: ["oops"]` (strings, not objects) must not crash `entry.get("sequence")`. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 1, "rejected": ["oops", 5, None]}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.rejected == [] + assert spool.watermark() == 1 + + +def test_events_client_503_503_404_does_not_raise_channel_failed(tmp_path: Path) -> None: + # N6: interleaved 5xx must not shorten the 404 budget -- only 3 OBSERVED 404s (not 3 attempts + # total) reach CHANNEL_FAILED. The ranked missing test names this sequence exactly. + # max_attempts=3 stops the retry loop right after this exact sequence (rather than letting it + # roll into a 4th, real-success attempt against FakePlatform, which would also prove the point + # but less precisely) -- classify_response must still read the single 404 as RETRYABLE, not + # CHANNEL_FAILED, because only ONE 404 has been observed. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient( + _capabilities(), spool, platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=3) + ) + + result = client.flush() # must NOT raise HostedChannelFailedError -- only one 404 seen so far + assert result.error is not None + assert result.error.outcome is ChannelOutcome.RETRYABLE + assert result.error.status_code == 404 + assert len(platform.calls) == 3 + + +def test_events_client_503_503_404_404_404_raises_channel_failed(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + for _ in range(3): + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedChannelFailedError) as excinfo: + client.flush() + assert excinfo.value.error.domain is FailureDomain.PLATFORM_SYNC + assert len(platform.calls) == 5 + + +def test_events_client_413_on_the_events_channel_does_not_forever_loop(tmp_path: Path) -> None: + # N7, ranked missing test 7: a 413 with no artifact_budget_exceeded body classifies as a + # PERMANENT_ITEM (not a nonsensical BUDGET_EXCEEDED on a non-artifact channel) so the batch does + # not silently re-send forever with no diagnostic. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(413, {"error": "request_entity_too_large", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is not None + assert result.error.outcome is ChannelOutcome.PERMANENT_ITEM + assert result.error.status_code == 413 + + +def test_events_client_413_halves_the_batch_and_retries(tmp_path: Path) -> None: + # N7: a 413 that DOES report artifact_budget_exceeded (unexpected on this channel, but the + # reactive halving does not care why) triggers a halve-and-retry rather than giving up outright. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(4): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + platform.queue(TransportResponse(413, {"error": "artifact_budget_exceeded", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + # First attempt (batch of 4) 413s; second attempt (batch of 2) succeeds. + assert result.acked_through_sequence == 2 + assert len(spool.pending_since_watermark()) == 2 + + +def test_events_client_channel_state_latches_a_fence_and_stops_touching_the_transport( + tmp_path: Path, +) -> None: + # N10: once fenced, EVERY subsequent call on a client sharing this ChannelState raises the SAME + # error immediately, without ever hitting the transport again. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + _spooled_log_event(spool, "event_y") + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + state = ChannelState() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, channel_state=state) + + with pytest.raises(HostedFencedError): + client.flush() + calls_after_first = len(platform.calls) + + with pytest.raises(HostedFencedError) as excinfo: + client.flush() + assert len(platform.calls) == calls_after_first # no new transport call at all + assert excinfo.value.error.code == "token_expired" + + +def test_channel_state_shared_across_clients_fences_all_of_them(tmp_path: Path) -> None: + # N10: the fence is per-ATTEMPT, not per-channel -- a fence observed on EventsClient must also + # stop a ResultsClient sharing the same ChannelState, with no transport call from the second. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(403, {"error": "fence_invalid", "message": "m", "retryable": False}, {})) + state = ChannelState() + events_client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, channel_state=state) + results_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, channel_state=state) + + with pytest.raises(HostedFencedError): + events_client.flush() + calls_before = len(platform.calls) + + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + with pytest.raises(HostedFencedError) as excinfo: + results_client.push(receipt) + assert len(platform.calls) == calls_before # results_client never touched the transport + assert excinfo.value.error.code == "fence_invalid" + + +def test_events_client_401_raises_hosted_fenced_error(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedFencedError) as excinfo: + client.flush() + assert excinfo.value.error.code == "token_expired" + assert spool.watermark() == 0 + + +def test_events_client_404_exhausted_three_times_raises_hosted_channel_failed_error(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + for _ in range(3): + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedChannelFailedError) as excinfo: + client.flush() + assert excinfo.value.error.domain is FailureDomain.PLATFORM_SYNC + assert len(platform.calls) == 3 + + +def test_events_client_crash_between_send_and_ack_redelivers_safely_within_one_flush(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_crash") + platform = _platform() + platform.crash_after_next_write = True + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == 1 + assert spool.watermark() == 1 + assert len(platform.calls) == 2 + assert "event_crash" in platform._events_by_id + + +def test_events_client_batch_size_is_clamped_to_the_contract_max(tmp_path: Path) -> None: + from fi.alk.harness.outbound import EVENTS_MAX_BATCH + + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(EVENTS_MAX_BATCH + 10): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, batch_size=10_000) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == EVENTS_MAX_BATCH + assert spool.watermark() == EVENTS_MAX_BATCH + # Everything past the batch cap is still pending for the next flush. + assert len(spool.pending_since_watermark()) == 10 + + +# --- ResultsClient ------------------------------------------------------------------------------- + + +def test_results_client_pushes_a_receipt_successfully() -> None: + platform = _platform() + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + result = client.push(receipt) + assert result.delivered + assert not result.already_existed + assert platform._receipts[("j1", "k")]["digest"] == receipt["digest"] + + +def test_results_client_permanent_item_is_never_retried() -> None: + platform = _platform() + platform.queue(TransportResponse(422, {"error": "artifact_unknown", "message": "m", "retryable": False}, {})) + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + + result = client.push(receipt) + assert not result.delivered + assert result.error is not None + assert result.error.outcome is ChannelOutcome.PERMANENT_ITEM + assert result.error.code == "artifact_unknown" + assert len(platform.calls) == 1 + + +def test_results_client_401_raises_hosted_fenced_error() -> None: + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + with pytest.raises(HostedFencedError): + client.push({"job_id": "j1", "scenario_key": "k", "digest": "sha256:" + "0" * 64, "attempt_number": 1}) + + +def test_results_client_crash_between_send_and_ack_redelivers_within_one_push() -> None: + platform = _platform() + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + platform.crash_after_next_write = True + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + + result = client.push(receipt) + assert result.delivered + assert result.already_existed # the retry loop's own second attempt saw the already-stored write + assert len(platform.calls) == 2 + + +def test_results_client_crash_between_send_and_ack_redelivers_across_a_simulated_restart() -> None: + platform = _platform() + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + platform.crash_after_next_write = True + # max_attempts=1: the retry loop can't self-heal, so the crash surfaces as a RETRYABLE return + # from this one push() call -- as if the guest process died right here. + first_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=1)) + first = first_client.push(receipt) + assert not first.delivered + assert first.error is not None and first.error.outcome is ChannelOutcome.RETRYABLE + + # A brand new client (simulating a fresh process) retries the same receipt: idempotent duplicate. + second_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + second = second_client.push(receipt) + assert second.delivered + assert second.already_existed + assert len(platform.calls) == 2 + + +def test_results_client_attempt_superseded_latches_the_shared_channel_state() -> None: + # N22: 409 attempt_superseded is item-level PERMANENT_ITEM for the ONE call that received it + # (contract-correct -- still returned normally, not raised), but folds into the N10 latch for + # every SUBSEQUENT call: "a fenced attempt's in-flight requests cannot land after registration + # of its successor" is a fence in substance. + platform = _platform() + platform.queue(TransportResponse(409, {"error": "attempt_superseded", "message": "m", "retryable": False}, {})) + state = ChannelState() + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, channel_state=state) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + + first = client.push(receipt) + assert not first.delivered + assert first.error is not None and first.error.code == "attempt_superseded" + + with pytest.raises(HostedAttemptSupersededError): + client.push(receipt) + assert len(platform.calls) == 1 # the second call never touched the transport + + +def test_build_event_record_redacts_a_dsn_in_world_unhealthy_cause() -> None: + # N9, ranked missing test 9: outbound-channels.md v1.3's redaction obligation + seams v1.11 + # §3's "any outbound projection ... redacts userinfo" example DSN shape, verbatim. + record = _event( + OutboundEventType.WORLD_UNHEALTHY, + HarnessStage.RUNNING, + {"world_index": 2, "cause": "connect failed: postgresql://harness:s3cr3t@localhost:14000/w0"}, + ) + cause = record["payload"]["cause"] + assert "s3cr3t" not in cause + assert cause == "connect failed: postgresql://harness:***@localhost:14000/w0" + # The digest must match the REDACTED bytes actually spooled, not the original with the secret. + assert record["digest"] == event_payload_digest(record["payload"]) + + +def test_build_event_record_redacts_terminal_failure_message() -> None: + record = _event( + OutboundEventType.TERMINAL, + HarnessStage.FAILED, + { + "stage": "failed", + "reason": None, + "failure": { + "domain": "infrastructure", + "stage": "running", + "code": "world_pool_exhausted", + "message": "dial postgresql://harness:hunter2@localhost:14000/w0: connection refused", + }, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + message = record["payload"]["failure"]["message"] + assert "hunter2" not in message + assert "postgresql://harness:***@localhost:14000/w0" in message + + +def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failure_message() -> None: + receipt = build_result_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid", + scenario_attempt=1, world_index=None, + status=ScenarioStatus.ERRORED, + sub_goals=[{"name": "n", "held": None, "reason": "leaked postgresql://harness:pw123@host/db", "judged": False}], + evaluations=[{"name": "n2", "kind": "metric", "score": 0.5, "reason": "saw postgresql://harness:pw123@host/db"}], + call=None, + failure={"domain": "agent", "stage": "running", "code": "c", "message": "postgresql://harness:pw123@host/db"}, + ) + assert "pw123" not in str(receipt) + assert receipt["sub_goals"][0]["reason"] == "leaked postgresql://harness:***@host/db" + assert receipt["evaluations"][0]["reason"] == "saw postgresql://harness:***@host/db" + assert receipt["failure"]["message"] == "postgresql://harness:***@host/db" + # The digest must match the redacted bytes. + ResultReceiptDraft.model_validate(receipt) # round-trips cleanly, digest agrees + + +def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: + assert redact_outbound_text("postgresql://harness:pw@localhost/db") == "postgresql://harness:***@localhost/db" + assert redact_outbound_text("no secrets here") == "no secrets here" + assert redact_outbound_text("token=abc123 leaked", extra_secret_values=("abc123",)) == "token=*** leaked" + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + # P3: the empty-username shape -- the canonical Redis/RabbitMQ/Mongo DSN form -- used to + # leave the password verbatim (the old pattern required a NON-empty username group). + ("redis://:secretpw@h:6379/0", "redis://:***@h:6379/0"), + # A password containing '@' must not truncate the mask at the first '@' seen. + ("mysql://u:p@ss@h/db", "mysql://u:***@h/db"), + # A bare token/username-only userinfo (no ':') is the standard shape a bearer token takes + # in git/registry output -- must be masked outright, not treated as "just a username." + ("https://ghp_TOKEN@github.com/o/r.git", "https://***@github.com/o/r.git"), + # Negative: a path segment containing '@' with no userinfo before it must be left alone. + ("https://example.com/a@b", "https://example.com/a@b"), + # Negative: no "://" at all -- not a userinfo shape. + ("mailto:a@b", "mailto:a@b"), + ], +) +def test_redact_outbound_text_handles_the_empty_username_and_bare_token_userinfo_shapes( + raw: str, expected: str +) -> None: + assert redact_outbound_text(raw) == expected + + +# --- ArtifactsClient ----------------------------------------------------------------------------- + + +def test_artifacts_client_rejects_a_locally_wrong_digest_before_any_request() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + with pytest.raises(ValueError, match="artifact_digest_mismatch_local"): + client.upload("0" * 64, b"hello world", kind=ArtifactKind.LOG) + assert platform.calls == [] + + +def test_artifacts_client_upload_new_then_already_exists() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"result contents" + digest_hex = hashlib.sha256(data).hexdigest() + + first = client.upload(digest_hex, data, kind=ArtifactKind.RESULT, scenario_key="k1") + assert first.delivered and not first.already_existed + second = client.upload(digest_hex, data, kind=ArtifactKind.RESULT, scenario_key="k1") + assert second.delivered and second.already_existed + + +def test_artifacts_client_digest_mismatch_gets_exactly_one_re_upload() -> None: + platform = _platform() + platform.queue(TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {})) + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"re-upload me" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert result.delivered + assert len(platform.calls) == 2 # the queued failure, then the one allowed re-upload + + +def test_artifacts_client_digest_mismatch_twice_is_permanent_the_scenario_is_errored() -> None: + platform = _platform() + for _ in range(2): + platform.queue(TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {})) + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"re-upload me twice" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert not result.delivered + assert result.error is not None and result.error.code == "digest_mismatch" + assert len(platform.calls) == 2 # exactly one re-upload attempted, per "re-upload once" + + +def test_artifacts_client_sends_x_artifact_size_derived_from_the_actual_bytes() -> None: + # "X-Artifact-Size (authoritative; mismatch -> 422 size_mismatch)": this client never accepts a + # caller-supplied size that could drift from the real payload -- it always derives the header + # from `len(data)` itself, so a size mismatch against what's actually sent is structurally + # unreachable from this client (the platform's own check remains the authority regardless). + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"exactly this many bytes" + digest_hex = hashlib.sha256(data).hexdigest() + + client.upload(digest_hex, data, kind=ArtifactKind.TRACE, scenario_key="k9") + assert platform.received_headers[-1]["X-Artifact-Size"] == str(len(data)) + assert platform.received_headers[-1]["X-Artifact-Kind"] == "trace" + assert platform.received_headers[-1]["X-Scenario-Key"] == "k9" + + +def test_artifacts_client_content_type_defaults_by_kind() -> None: + # N17: recordings mp4, transcript a JSON array (per §3a "Formats") -- not a blanket octet-stream. + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"recording bytes" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.RECORDING_COMBINED) + assert platform.received_headers[-1]["Content-Type"] == "video/mp4" + + data2 = b"transcript bytes" + digest2 = hashlib.sha256(data2).hexdigest() + client.upload(digest2, data2, kind=ArtifactKind.TRANSCRIPT) + assert platform.received_headers[-1]["Content-Type"] == "application/json" + + data3 = b"trace bytes" + digest3 = hashlib.sha256(data3).hexdigest() + client.upload(digest3, data3, kind=ArtifactKind.TRACE) + assert platform.received_headers[-1]["Content-Type"] == "application/octet-stream" + + +def test_artifacts_client_content_type_override() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"custom type" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.OTHER, content_type="text/plain") + assert platform.received_headers[-1]["Content-Type"] == "text/plain" + + +def test_artifacts_client_latches_after_413_and_skips_non_reserved_without_the_transport() -> None: + # N18: once a 413 artifact_budget_exceeded is observed, later non-reserved uploads must be + # refused LOCALLY (never touching the transport again); reserved kinds keep going. + platform = _platform() + platform.budget_remaining = 1 + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + first_data = b"too big for the tiny budget" + first = client.upload(hashlib.sha256(first_data).hexdigest(), first_data, kind=ArtifactKind.TRACE) + assert first.error is not None and first.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + calls_after_413 = len(platform.calls) + + second_data = b"another non-reserved upload" + second = client.upload(hashlib.sha256(second_data).hexdigest(), second_data, kind=ArtifactKind.LOG) + assert not second.delivered + assert second.error is not None and second.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert len(platform.calls) == calls_after_413 # no new transport call + + reserved_data = b"reserved kind always goes through" + reserved_digest = hashlib.sha256(reserved_data).hexdigest() + third = client.upload(reserved_digest, reserved_data, kind=ArtifactKind.RESULT) + assert third.delivered # reserved kinds are never latched out + assert len(platform.calls) == calls_after_413 + 1 + + +def test_artifacts_client_413_budget_exceeded_is_never_retried() -> None: + platform = _platform() + platform.budget_remaining = 5 + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"too big for the budget" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert not result.delivered + assert result.error is not None + assert result.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert len(platform.calls) == 1 + + +def test_artifacts_client_chunks_uploads_over_the_threshold_and_reassembles_correctly() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None, chunk_threshold_bytes=10, chunk_size_bytes=4) + data = b"0123456789abcdef" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.TRACE) + assert result.delivered + assert platform._artifacts[digest_hex] == data + + +def test_artifacts_client_crash_between_send_and_ack_redelivers_as_already_exists() -> None: + platform = _platform() + data = b"artifact bytes" * 100 + digest_hex = hashlib.sha256(data).hexdigest() + platform.crash_after_next_write = True + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + + result = client.upload(digest_hex, data, kind=ArtifactKind.TRACE) + assert result.delivered + assert result.already_existed + assert len(platform.calls) == 2 + assert platform._artifacts[digest_hex] == data + + +def test_artifacts_client_push_manifest_is_idempotent_and_checks_references() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"manifest target" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.RESULT) + + manifest = build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": f"sha256:{digest_hex}", "kind": "result", "size": len(data), "scenario_key": "k"}], + complete=True, + ) + first = client.push_manifest(manifest) + assert first.delivered and not first.already_existed + second = client.push_manifest(manifest) + assert second.delivered and second.already_existed # idempotent on (attempt_id, digest) + + +def test_artifacts_client_push_manifest_rejects_an_unknown_artifact_reference() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + manifest = build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": "sha256:" + "f" * 64, "kind": "result", "size": 1, "scenario_key": "k"}], + complete=True, + ) + result = client.push_manifest(manifest) + assert not result.delivered + assert result.error is not None and result.error.code == "artifact_unknown" From 43ee1513a9dac883bfdf250f9de24f77859fb738 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 15:02:53 +0530 Subject: [PATCH 07/20] =?UTF-8?q?feat(harness):=20hosted=20scheduler=20?= =?UTF-8?q?=E2=80=94=20world=20pool,=20scenario=20loop,=20retry,=20receipt?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scheduler that drains a job's scenarios across W provisioned worlds per seam contract v1.12: the world pool with §4.5b-serialized provider calls, degrade-tolerant start, pool-owned world_unhealthy emission and the §5.4 zero-ready grace rule; per-scenario reset-lease-run-grade with the v3.4 receipt vocabulary, retry-once on another world, exact skipped synthesis, and cancel awareness; scenario phases dispatched on a dedicated executor so a leaked thread can never starve the provider. Three cold review rounds, closed by a 23-mutant mutation run; survivors (S1–S10, KD-1–6) recorded in .claude/harness-alk/reports/p9-review-r3.md. Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_scheduler.py | 1494 +++++++++++++++++++++++ tests/harness/test_hosted_scheduler.py | 1555 ++++++++++++++++++++++++ 2 files changed, 3049 insertions(+) create mode 100644 src/fi/alk/harness/hosted_scheduler.py create mode 100644 tests/harness/test_hosted_scheduler.py diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py new file mode 100644 index 00000000..3991723c --- /dev/null +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -0,0 +1,1494 @@ +"""World pool + scenario loop — `hosted-execution-seams.md` v1.12 §4/§5, `world-handle-interface.md` +v3.4, `outbound-channels.md` v1.3. + +Owns: leasing/releasing the W worlds `process_runtime.ProcessRuntimeProvider.provision()` hands +back, resetting a world to pristine before each scenario (spine §4.2), running one scenario's +`setup`/`ready`/checks against the world handle (the return-convention + errored-receipt table in +`world-handle-interface.md`), the fixed one-retry-on-a-fresh-world rule (spine §5 step 4), and +synthesizing a complete receipt ledger (one per scenario, `skipped` for anything never attempted). + +Decoupling, deliberate: +- `process_runtime.py` is the real, settled provisioner — imported directly (`EnvironmentRuntime`, + `RuntimeState`). `WorldProvisioner` below is a structural `Protocol` matching + `ProcessRuntimeProvider`'s actual async shape so tests can inject a fake without touching a real + filesystem/subprocess tree. +- `outbound.py` is being written in parallel and its surface is not pinned yet, so nothing here + imports it. `OutboundPort` is this module's own minimal sink for the events/receipts it + produces, typed against `outbound-channels.md`'s closed vocabulary; whoever wires the real + client adapts to it. +- The Scenario Generation Contract (Karthik, in review) is not available here either, so `Scenario` + is this module's own minimal Protocol for what the loop needs: a key/id pair, `setup`/`ready`, + and named sub-goal checks. Same for the simulated "call" itself (a different track's seam) — + `CallRunner` is injected. +- Secrets and the cancel signal are entrypoint-owned (P10); `cancel_requested` is an injected + zero-argument callable. +""" + +from __future__ import annotations + +import asyncio +import inspect +import random +import re +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Awaitable, Callable, Protocol, Sequence + +from .job import FailureDomain, HarnessStage +from .process_runtime import EnvironmentRuntime, RuntimeState +from .world.errors import ( + WorldError, + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from .world.runtime import Call + +# --- the World handle (world-handle-interface.md v3.4) -------------------------------------- +# +# The frozen contract's code block gives six verbs plus `world_index`/`rng`. `read_only()` is not +# in that block, but the contract still requires `ready`/`check` to receive a handle whose writes +# raise `WorldReadOnly` (own section, "Read-only handles") without saying how a caller gets one — +# the shipped `HostedWorld.read_only()` (`world/handle.py`) already names this exact operation, so +# mirroring it here is the reversible choice: a real `HostedWorld` satisfies this Protocol as-is. +# +# m8: `World.read_only()` used to be typed `-> "World"`, but the real `ReadOnlyWorld` it returns +# has no `read_only()` of its own (mirroring `world/handle.py`'s own `ReadOnlyWorld`, which is +# deliberately not re-enterable) — so it fails a structural check against `World` itself. +# `ReadOnlyWorld` below names the narrower surface `ready`/`check` actually receive. + + +class ReadOnlyWorld(Protocol): + world_index: int + rng: random.Random + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: ... + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: ... + + def change( + self, collection: str, key: str, changes: dict[str, Any], *, by: str = "" + ) -> int: ... + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: ... + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: ... + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: ... + + +class World(Protocol): + world_index: int + rng: random.Random + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: ... + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: ... + + def change( + self, collection: str, key: str, changes: dict[str, Any], *, by: str = "" + ) -> int: ... + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: ... + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: ... + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: ... + + def read_only(self) -> ReadOnlyWorld: ... + + +class WorldFactory(Protocol): + """Builds the `World` handle for one already-reset `EnvironmentRuntime`. + + Deliberately not this module's job: `HostedWorld` needs a `PostgresStore` (parsed from the + runtime's `database` endpoint) plus the baseline row counts the provisioner measured at + freeze time — both live behind `ProcessRuntimeProvider`'s private state, which §4's + `RuntimeProvider` Protocol never exposes. Injected instead of guessed. + """ + + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> World: ... + + +# --- the provisioner surface this module actually drives ------------------------------------- +# +# Matches `ProcessRuntimeProvider`'s real async shape (process_runtime.py) structurally, not the +# older single-runtime `runtime.RuntimeProvider`. `bundle` stays `Any` — this module never reads a +# bundle field itself, only threads it back into `provision()` reconcile calls, so it does not +# need `EnvironmentBundleV2`'s own in-flux-adjacent type. +# +# M1 (spine v1.12 §4): `bundle_dir` is a required keyword — §2c seed/migration paths resolve +# against the verified bundle root, never against `source`. `require_declared_user` is dropped +# entirely: it is not in §4's signature, and the real provider now defaults it `True` on its own +# (the local lane opts out at provider construction, not per call). +# M2 (spine §4 point 3): `healthy` — declared readiness probes, not "process is running" — is a +# port method, not an optionally-injected callable. + + +class WorldProvisioner(Protocol): + async def provision( + self, + bundle: Any, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, + contract: Any | None = None, + instances: int = 1, + ) -> list[EnvironmentRuntime]: ... + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: ... + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: ... + + async def close(self, *, work_directory: Path) -> None: ... + + +# --- scenarios (this module's own minimal surface; Karthik's contract is not wired yet) ------- + + +class SubGoal(Protocol): + name: str + judged: str # `sub_goals[].judged` per outbound-channels.md: boolean is `judged != ""`. + + def check(self, world: ReadOnlyWorld, calls: Sequence[Call]) -> object: ... + + +class Scenario(Protocol): + scenario_key: str + scenario_id: str # platform id from pre-allocation (outbound-channels.md Channel 2 "Join"). + sub_goals: Sequence[SubGoal] + + def setup(self, world: World) -> object: ... + + def ready(self, world: ReadOnlyWorld) -> object: ... + + +# --- the simulated call (a different track's seam; injected, never built here) --------------- + + +@dataclass(frozen=True) +class CallOutcome: + calls: tuple[Call, ...] + turns: int + started_at: str | None + ended_at: str | None + duration_ms: int + transcript_artifact: str | None = None + recording_artifacts: tuple[str, ...] = () + + +class CallAborted(RuntimeError): + """The call step started but did not finish. `partial`, when known, carries whatever timing + the call runner already measured — the receipt's `call` field must not be null once the call + has genuinely started (outbound-channels.md Channel 2, "errored receipt body").""" + + def __init__(self, message: str, *, partial: CallOutcome | None = None) -> None: + super().__init__(message) + self.partial = partial + + +class CallRunner(Protocol): + async def run(self, scenario: Scenario, runtime: EnvironmentRuntime) -> CallOutcome: ... + + +# --- receipts (outbound-channels.md Channel 2; envelope fields — job_id/attempt_id/digest/etc — +# are the emitter's concern, not reproduced here) ----------------------------------------------- + + +@dataclass(frozen=True) +class SubGoalResult: + name: str + held: bool | None + reason: str | None + judged: bool + + +@dataclass(frozen=True) +class Evaluation: + name: str + kind: str # "metric" | "checkpoint" + reason: str + score: float | None = None + passed: bool | None = None + + +@dataclass(frozen=True) +class CallSummary: + started_at: str | None + ended_at: str | None + duration_ms: int + turns: int + transcript_artifact: str | None = None + recording_artifacts: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ReceiptFailure: + domain: str + stage: str + code: str + message: str + + +@dataclass(frozen=True) +class ResultReceipt: + scenario_key: str + scenario_id: str + scenario_attempt: int + world_index: int | None + status: str # "passed" | "failed" | "errored" | "skipped" + sub_goals: tuple[SubGoalResult, ...] + evaluations: tuple[Evaluation, ...] + call: CallSummary | None + failure: ReceiptFailure | None + + +# --- outbound (this module's own minimal sink; see the module docstring's decoupling note) ---- + + +class OutboundPort(Protocol): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: ... + + async def scenario_retried( + self, *, scenario_key: str, from_world: int, to_world: int + ) -> None: ... + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: ... + + async def log(self, *, level: str, message: str) -> None: ... + + async def receipt(self, receipt: ResultReceipt) -> None: ... + + +# --- failure-code -> FailureDomain, and which codes retry once on a fresh world ---------------- +# +# `world_unavailable` is domain `environment` per world-handle-interface.md's own errored-receipt +# table (it overrides the table's default "domain: simulator"). `evidence_missing` is domain +# simulator but is explicitly carved out as retryable in that same document ("gets the same single +# retry-on-another-world as a world failure"). `call_failed` (v3.3) and `driver_crashed` (v3.4) are +# both rows in that same closed table now: `call_failed` is domain infrastructure, retried once +# like a world failure; `driver_crashed` is domain simulator, not retried — the scheduler's own +# machinery failing while driving a scenario, distinct from any agent/check/call outcome. +# `world_pool_exhausted` is NOT a per-scenario receipt code — it is `HostedScheduler.run()`'s own +# job-abort signal for spine v1.12 §5.4's closed job-level failure vocabulary for stage `running` +# (domain infrastructure). + +_CODE_DOMAIN: dict[str, FailureDomain] = { + "setup_crashed": FailureDomain.SIMULATOR, + "setup_timeout": FailureDomain.SIMULATOR, + "ready_timeout": FailureDomain.SIMULATOR, + "check_timeout": FailureDomain.SIMULATOR, + "ready_not_ready": FailureDomain.SIMULATOR, + "ready_broken": FailureDomain.SIMULATOR, + "check_broken": FailureDomain.SIMULATOR, + "evidence_missing": FailureDomain.SIMULATOR, + "world_usage": FailureDomain.SIMULATOR, + "world_unavailable": FailureDomain.ENVIRONMENT, + "state_too_large": FailureDomain.SIMULATOR, + "call_failed": FailureDomain.INFRASTRUCTURE, + "driver_crashed": FailureDomain.SIMULATOR, + "world_pool_exhausted": FailureDomain.INFRASTRUCTURE, +} +_RETRYABLE_CODES = frozenset({"evidence_missing"}) + +# M13: an exception/overrun outcome leaves the world half-applied — world-handle-interface.md's +# return-conventions rule is "the world is discarded and re-provisioned (a half-applied world is +# never reused)." `ready_not_ready` is deliberately excluded: a precondition failing on the shared +# sealed baseline is a clean verdict, not an exception, so the world itself is still fine. +_DISCARD_ON_ERROR_CODES = frozenset( + { + "setup_crashed", + "setup_timeout", + "ready_timeout", + "check_timeout", + "ready_broken", + "check_broken", + "world_usage", + "state_too_large", + } +) + +SETUP_TIMEOUT_SECONDS = 60.0 +READY_TIMEOUT_SECONDS = 15.0 +CHECK_TIMEOUT_SECONDS = 60.0 + +_MESSAGE_LIMIT = 2000 # matches the Call.result/error truncation convention (world-handle-interface.md). +_CAUSE_LIMIT = 200 # outbound-channels.md Channel 1: `world_unhealthy.cause` free text <=200. +_USERINFO_PATTERN = re.compile(r"://[^@/]+@") + + +def _is_retryable(code: str) -> bool: + return _CODE_DOMAIN[code] in (FailureDomain.ENVIRONMENT, FailureDomain.INFRASTRUCTURE) or ( + code in _RETRYABLE_CODES + ) + + +def _truncate(text: str, limit: int = _MESSAGE_LIMIT) -> str: + if len(text) <= limit: + return text + return text[: limit - len("…[truncated]")] + "…[truncated]" + + +def _sanitize_cause(message: str) -> str: + # M7: `cause` is capped at 200 chars and must never carry endpoint credentials — postgres + # error strings routinely embed the DSN (`postgresql://user:pw@host/db`). + return _truncate(_USERINFO_PATTERN.sub("://***@", message), _CAUSE_LIMIT) + + +def _failure(code: str, message: str) -> ReceiptFailure: + return ReceiptFailure( + domain=_CODE_DOMAIN[code].value, + stage=HarnessStage.RUNNING.value, + code=code, + message=_truncate(message), + ) + + +# --- return-convention classification (world-handle-interface.md "Return conventions") -------- + + +@dataclass(frozen=True) +class _Verdict: + held: bool + reason: str | None + broken: bool + + +def _classify_ready(value: object) -> _Verdict: + if value is None or value is True: + return _Verdict(True, None, False) + if isinstance(value, str): + if value.strip() == "": + return _Verdict(True, None, False) + return _Verdict(False, value, False) + # Bare False or any other value -> broken. checks.py's `run_world_check` treats a non-None, + # non-string ready() answer the same way; a scenario hitting this cannot be told apart from a + # buggy ready.py, which is why it is `ready_broken` rather than a clean not-ready verdict. + return _Verdict(False, None, True) + + +def _classify_check(value: object) -> _Verdict: + if value is None or value is True: + return _Verdict(True, None, False) + if isinstance(value, str): + if value.strip() == "": + return _Verdict(True, None, False) + return _Verdict(False, value, False) + if value is False: + # An agent result ("the agent did something wrong"), not a broken check — matches + # checks.py's `Outcome(name, False, "False")`. + return _Verdict(False, "False", False) + return _Verdict(False, None, True) + + +# --- phase execution: budget + exception classification --------------------------------------- + + +class _PhaseTimeout(Exception): + def __init__(self, phase: str) -> None: + super().__init__(phase) + self.phase = phase + + +class _PhaseNeverStarted(Exception): + """R1: the phase's own worker thread had not even started running when its budget elapsed — + the dedicated executor was saturated, not the phase itself overrunning. Must not read as a + genuine timeout (which discards the world); the world did nothing wrong here.""" + + def __init__(self, phase: str) -> None: + super().__init__(phase) + self.phase = phase + + +class _PhaseWorldGone(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseMisuse(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseStateTooLarge(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseCrashed(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +async def _invoke( + fn: Callable[..., object], + *args: object, + timeout: float, + phase: str, + executor: ThreadPoolExecutor, +) -> object: + # R1: a `threading.Event` set as the thread body's first statement — the only way to tell + # "the phase ran past its budget" (genuine overrun, world half-applied) apart from "the + # phase's thread was still queued behind others when the budget elapsed" (the scheduler's own + # executor was saturated; the world itself never touched anything). + started_flag = threading.Event() + + def _run() -> object: + started_flag.set() + return fn(*args) + + async def _call() -> object: + # B4: real scenario code (`setup`/`ready`/`check`) is synchronous, blocking psycopg calls + # — it must never run directly on the event loop, or the timeout below is purely + # decorative and every other world stalls with it. Dispatched to the scheduler's own + # dedicated executor (world-handle-interface.md: "one worker thread per world"; R1 — + # never the loop's default executor, which the provider's own `to_thread` calls also + # use). If the thread's own return value is itself awaitable (scenario code that is + # `async def`, reached indirectly through a sync wrapper), that coroutine is driven on + # the event loop afterward, where real suspension/cancellation actually works — this is + # the kept "awaitable" branch. + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(executor, _run) + if inspect.isawaitable(result): + result = await result + return result + + loop = asyncio.get_running_loop() + started = loop.time() + try: + return await asyncio.wait_for(_call(), timeout=timeout) + except asyncio.TimeoutError as exc: + # m4: `asyncio.TimeoutError is TimeoutError` on 3.11 — a psycopg statement timeout raised + # INSIDE `fn` looks identical to `wait_for`'s own deadline unless the elapsed time is + # actually checked. If the budget did not genuinely elapse, this was `fn`'s own timeout + # bubbling through — a broken phase, not a budget overrun. + if loop.time() - started < timeout: + raise _PhaseCrashed(phase, exc) from exc + if not started_flag.is_set(): + raise _PhaseNeverStarted(phase) from exc + # B4: `wait_for`'s cancellation stops US from waiting on the thread, not the thread + # itself — psycopg in-flight cancellation is not wired here (P11 follow-up; recorded in + # the fixer report). The thread is abandoned, bounded by scenario count per the contract's + # own accepted tradeoff; its world is discarded rather than reused (M13). + raise _PhaseTimeout(phase) from exc + except WorldUnavailable as exc: + raise _PhaseWorldGone(phase, exc) from exc + except WorldStateTooLarge as exc: + raise _PhaseStateTooLarge(phase, exc) from exc + except (WorldReadOnly, WorldReservedName, WorldQueryRejected, WorldUsageError) as exc: + raise _PhaseMisuse(phase, exc) from exc + except WorldError as exc: + # m5: catches any WorldError subclass not special-cased above (world/errors.py's own + # base, kept exactly for "route 'scenario code misused the handle' to one outcome without + # naming all six") — a future seventh subclass lands here instead of silently falling into + # the generic crash classification below. + raise _PhaseMisuse(phase, exc) from exc + except Exception as exc: + raise _PhaseCrashed(phase, exc) from exc + + +_CRASH_CODE_BY_PHASE = {"setup": "setup_crashed", "ready": "ready_broken", "check": "check_broken"} +_TIMEOUT_CODE_BY_PHASE = {"setup": "setup_timeout", "ready": "ready_timeout", "check": "check_timeout"} + + +@dataclass(frozen=True) +class _PhaseResult: + value: object + failure: ReceiptFailure | None + + +async def _run_phase( + fn: Callable[..., object], + *args: object, + timeout: float, + phase: str, + executor: ThreadPoolExecutor, +) -> _PhaseResult: + try: + value = await _invoke(fn, *args, timeout=timeout, phase=phase, executor=executor) + return _PhaseResult(value, None) + except _PhaseNeverStarted: + # R1: not the phase's fault and not the world's — the scheduler's own thread pool + # couldn't service it in time. `driver_crashed` is not in `_DISCARD_ON_ERROR_CODES`, so + # this releases the world rather than discarding a perfectly healthy one. + return _PhaseResult( + None, + _failure( + "driver_crashed", f"{phase} never started before its budget elapsed (thread pool saturated)" + ), + ) + except _PhaseTimeout: + return _PhaseResult(None, _failure(_TIMEOUT_CODE_BY_PHASE[phase], f"{phase} exceeded its budget")) + except _PhaseWorldGone as exc: + return _PhaseResult(None, _failure("world_unavailable", str(exc.cause))) + except _PhaseMisuse as exc: + return _PhaseResult(None, _failure("world_usage", str(exc.cause))) + except _PhaseStateTooLarge as exc: + return _PhaseResult(None, _failure("state_too_large", str(exc.cause))) + except _PhaseCrashed as exc: + return _PhaseResult( + None, _failure(_CRASH_CODE_BY_PHASE[phase], f"{type(exc.cause).__name__}: {exc.cause}") + ) + + +# --- the world pool ----------------------------------------------------------------------------- + + +class NoWorldsAvailable(RuntimeError): + """Every provisioned world is down and none is currently recoverable (spine v1.12 §5.4: + "if ready worlds reach 0 the job FAILS in stage running, domain infrastructure" — declared + only after in-flight re-provisioning completes without restoring a world), OR the pool has + been closed (R5: `reason="closed"`).""" + + def __init__(self, message: str, *, reason: str = "exhausted") -> None: + super().__init__(message) + self.reason = reason + + +_RECONCILE_MAX_ATTEMPTS = 3 +_RECONCILE_BACKOFF_SECONDS = (0.05, 0.1) +_LEASE_POLL_INTERVAL_SECONDS = 0.02 +_CLOSE_RECONCILE_WAIT_SECONDS = 30.0 # R4: bounded wait for an in-flight reconcile before close() +# falls back to cancelling it (which cannot stop a thread-backed provider call already running). + + +class WorldPool: + """Leases/releases the W worlds `provisioner.provision()` returns, resets one to pristine on + every lease (spine §4.2), and reconciles an unhealthy world back in via `provision()` again + (§4 rule 1: "a sick world mid-job is recovered by calling `provision` again") — in the + background, so a lease elsewhere never blocks on someone else's recovery. + + B1/B2/M6 (spine v1.12 §4.5b): the provider port is NOT reentrant — at most one + `provision`/`reset`/`healthy`/`close` call is ever in flight, serialized by `_provider_lock` + (R13: `healthy` writes — it demotes state — so v1.12 folded it into the same serialized set + that provision/reset/close were already in; it is no longer treated as a read-only probe + exempt from the lock). A demotion that lands while a reconcile is already running is coalesced + into a trailing pass rather than a second concurrent `provision()` call. + """ + + def __init__( + self, + provisioner: WorldProvisioner, + *, + bundle: Any, + source: Path, + bundle_dir: Path, + work_directory: Path, + instances: int, + outbound: OutboundPort | None = None, + ) -> None: + self._provisioner = provisioner + self._bundle = bundle + self._source = source + self._bundle_dir = bundle_dir + self._work_directory = work_directory + self._instances = instances + self._outbound = outbound + + self._runtimes: dict[int, EnvironmentRuntime] = {} + self._available: set[int] = set() + self._leased: set[int] = set() + self._down: set[int] = set() + self._fresh: set[int] = set() # m9: provisioned/recovered but never yet leased/reset + self._effective_size = 0 # R2: the achieved world count `start()` settled on + + # m1: `asyncio.Condition` (not a manual `Event` + `clear()`) — waiting and notifying share + # one lock, so there is no window between releasing a lock and clearing a flag for a + # `set()` to land in and be silently lost. + self._state_lock = asyncio.Condition() + self._provider_lock = asyncio.Lock() + self._reconcile_task: asyncio.Task[None] | None = None + self._reconcile_pending = False + self._started = False + self._closing = False # R4: set at the top of close() -- lets an in-flight reconcile bail + # between attempts instead of burning close()'s wait budget on a pool being torn down. + self._closed = False # R5: set once close() has actually run -- latches provision()/lease() + # out for good; close() itself becomes idempotent. + + @property + def effective_size(self) -> int: + """R2: the world count `start()` actually achieved — may be less than the requested + `instances` on a legitimate degrade (conformance-gate failure, `fixed_port`). P10 sizes + `parallelism_degraded` and anything else that needs "how many worlds do we really have" + off this, never off the originally requested `instances`.""" + return self._effective_size + + @property + def size(self) -> int: + return len(self._runtimes) + + async def start(self) -> list[EnvironmentRuntime]: + if self._started: + # m10: a second call would re-provision behind every already-leased world's back. + raise RuntimeError("WorldPool.start() called more than once") + self._started = True + + async with self._provider_lock: + runtimes = await self._provisioner.provision( + self._bundle, + source=self._source, + bundle_dir=self._bundle_dir, + work_directory=self._work_directory, + instances=self._instances, + ) + + # R2 (spine v1.12 §4's conformance gate / `fixed_port`): `provision()` legitimately + # returns FEWER than `instances` worlds — "Fail → effective parallelism 1 + + # parallelism_degraded ... Loud, never silent," not a failure this pool should raise on. + # Reject only a genuinely malformed result: zero worlds, duplicates, a non-contiguous + # index set, or more worlds than were ever requested. + indices = {runtime.world_index for runtime in runtimes} + if ( + not runtimes + or len(runtimes) != len(indices) + or indices != set(range(len(runtimes))) + or len(runtimes) > self._instances + ): + # m10/R2: spine §4 — "ordered by world_index" and contiguous from 0 (what + # `range(effective_instances)` on the provider side guarantees). + raise RuntimeError( + f"provision() returned world_index set {sorted(indices)}, expected a contiguous " + f"0..N-1 subset of 0..{self._instances - 1}" + ) + self._effective_size = len(runtimes) + + async with self._state_lock: + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + if runtime.state in (RuntimeState.READY, RuntimeState.PREPARING): + # m10: never hand out a world provision() itself returned UNHEALTHY. A + # PREPARING world legitimately demotes straight to UNHEALTHY on a failed first + # reset/probe (spine v1.12 §3's preparing->unhealthy transition) -- lease()'s + # own health gate covers that case; nothing extra is needed here. + self._available.add(runtime.world_index) + if runtime.state is RuntimeState.READY: + self._fresh.add(runtime.world_index) + else: + self._down.add(runtime.world_index) + self._state_lock.notify_all() + return runtimes + + def _reconcile_in_flight(self) -> bool: + return self._reconcile_task is not None and not self._reconcile_task.done() + + async def _wait_bounded(self, *, poll: bool) -> None: + if not poll: + await self._state_lock.wait() + return + try: + await asyncio.wait_for(self._state_lock.wait(), timeout=_LEASE_POLL_INTERVAL_SECONDS) + except asyncio.TimeoutError: + pass # `Condition.wait()` reacquires the lock before propagating even on timeout. + + async def lease( + self, *, exclude: frozenset[int] = frozenset(), abandon: Callable[[], bool] | None = None + ) -> tuple[int, EnvironmentRuntime] | None: + """Returns `None` if `abandon()` reports true while this call was queued (B5) — the caller + never received a world, so there is nothing to release.""" + while True: + # R5: latched once close() has run — a lease past that point must never spawn a + # `reset()`/`healthy()` call against a provider that may already be hard-cleaned. + if self._closed: + raise NoWorldsAvailable("world pool is closed", reason="closed") + if abandon is not None and abandon(): + return None + + async with self._state_lock: + candidates = self._available - exclude + if candidates: + world_index = min(candidates) + self._available.discard(world_index) + self._leased.add(world_index) + skip_reset = world_index in self._fresh # m9 + self._fresh.discard(world_index) + else: + # Not just "every world is down" (the plain retry-exhausted case) — a world + # excluded for this lease (a same-scenario retry avoiding its failed world) + # can never satisfy `candidates` again no matter how long we wait, so it must + # count as unusable here too or a single-world pool's retry blocks forever. + usable = set(self._runtimes) - exclude + if not (usable - self._down): + # M9/R10 (spine v1.12 §5.4): declare exhaustion only once no reconcile is + # in flight or about to be — never on an instantaneous snapshot of world + # states. `_reconcile_pending` (set inside `mark_unhealthy`'s own critical + # section, R10) covers the gap between a demotion and its reconcile task + # actually existing. + if self._reconcile_in_flight() or self._reconcile_pending: + await self._wait_bounded(poll=abandon is not None) + continue + raise NoWorldsAvailable( + f"{len(self._down)}/{len(self._runtimes)} worlds unhealthy, " + f"none available outside {sorted(exclude)}" + ) + await self._wait_bounded(poll=abandon is not None) + continue + + reset_exc: Exception | None = None + probed_runtime: EnvironmentRuntime | None = None + if not skip_reset: + async with self._provider_lock: + runtime = self._runtimes.get(world_index) + probed_runtime = runtime + if runtime is not None: + try: + await self._provisioner.reset(runtime, work_directory=self._work_directory) + except Exception as exc: # noqa: BLE001 - B3: must never leak out of lease() + reset_exc = exc + + is_healthy = False + if reset_exc is None: + # M2: `healthy()` is called unconditionally after reset — including the m9 fast + # path, which skips only the (expensive) reset call, never the readiness check. + # R13 (spine v1.12 §4.5b): `healthy` now rides the port's non-reentrancy rule too, + # so it goes under `_provider_lock` like reset/provision/close. + async with self._provider_lock: + runtime = self._runtimes.get(world_index) + probed_runtime = runtime + if runtime is not None: + try: + is_healthy = await self._provisioner.healthy( + runtime, work_directory=self._work_directory + ) + except Exception as exc: # noqa: BLE001 + reset_exc = exc + + async with self._state_lock: + # m2/R14: re-read after the awaited provider calls — a concurrent reconcile may + # have replaced or dropped this index's `EnvironmentRuntime` while lease() awaited. + # `is_healthy` was computed against `probed_runtime` specifically; if the object + # at this index is no longer that same object, the verdict no longer describes it + # — discard this attempt and let the outer loop re-evaluate the index fresh rather + # than apply a stale verdict to a new object. + runtime = self._runtimes.get(world_index) + if runtime is None or runtime is not probed_runtime: + self._leased.discard(world_index) + self._state_lock.notify_all() + continue + if is_healthy and runtime.state is RuntimeState.READY: + self._state_lock.notify_all() + return world_index, runtime + cause = ( + f"reset failed: {reset_exc}" + if reset_exc is not None + else f"reset left world in state {runtime.state.value}" + ) + + await self.mark_unhealthy(world_index, cause=cause) + # loop again — this index is now excluded via `_down`, no explicit retry bookkeeping. + + async def release(self, world_index: int) -> None: + async with self._state_lock: + self._leased.discard(world_index) + if world_index in self._runtimes and world_index not in self._down: + self._available.add(world_index) + self._state_lock.notify_all() + + async def mark_unhealthy(self, world_index: int, *, cause: str) -> None: + async with self._state_lock: + self._leased.discard(world_index) + self._available.discard(world_index) + self._fresh.discard(world_index) + self._down.add(world_index) + runtime = self._runtimes.get(world_index) + if runtime is not None: + # M12 (spine v1.12 §4.5b, normative): the scheduler demotes `state` on the + # provider's own live `EnvironmentRuntime` object — that demotion is the signal + # the NEXT `provision()` reconciles on. + runtime.state = RuntimeState.UNHEALTHY + # R10: set inside this same critical section (not left to `_schedule_reconcile`'s own, + # later one) so a `lease()` observing state in the gap between the two never sees + # "every world down, no reconcile in flight or pending" and raises spuriously. + self._reconcile_pending = True + self._state_lock.notify_all() + + # R6: this is the sole path every demotion (this method) goes through, so it is the one + # place `world_unhealthy` needs to be emitted from for all four call sites to get it. + if self._outbound is not None: + try: + await self._outbound.world_unhealthy(world_index=world_index, cause=_sanitize_cause(cause)) + except Exception as exc: # noqa: BLE001 - B3: outbound failures are never fatal. + await self._log(f"world_unhealthy emit failed: {exc}") + + await self._schedule_reconcile() + + async def _schedule_reconcile(self) -> None: + async with self._state_lock: + if self._closed: + return # R5: never spawn new provider work once the pool has been closed. + if self._reconcile_in_flight(): + # B1/M6: a demotion landing mid-reconcile is coalesced into a trailing pass + # (`_reconcile_loop`) rather than a second concurrent `provision()` call. + self._reconcile_pending = True + return + self._reconcile_task = asyncio.create_task(self._reconcile_loop()) + + async def _reconcile_loop(self) -> None: + while True: + async with self._state_lock: + self._reconcile_pending = False + await self._reconcile() + async with self._state_lock: + if not self._reconcile_pending: + return + + async def _reconcile(self) -> None: + # M5: bounded retry with backoff — a single transient `provision()` failure (a momentary + # ENOSPC, an engine hiccup) used to retire its world for the rest of the job with no + # signal anywhere. Every failed attempt is logged through `OutboundPort` (when wired), + # matching the contract's own "loud, never silent" standard for degradation. + runtimes: list[EnvironmentRuntime] | None = None + last_exc: Exception | None = None + for attempt in range(1, _RECONCILE_MAX_ATTEMPTS + 1): + if self._closing: + # R4: close() is already bounded-waiting on this task — do not spend its wait + # budget retrying a pool that is being torn down anyway. + return + try: + async with self._provider_lock: + runtimes = await self._provisioner.provision( + self._bundle, + source=self._source, + bundle_dir=self._bundle_dir, + work_directory=self._work_directory, + instances=self._instances, + ) + except Exception as exc: # noqa: BLE001 - a reconcile must never crash the pool + last_exc = exc + await self._log( + f"world pool reconcile attempt {attempt}/{_RECONCILE_MAX_ATTEMPTS} failed: {exc}" + ) + if attempt < _RECONCILE_MAX_ATTEMPTS and not self._closing: + await asyncio.sleep(_RECONCILE_BACKOFF_SECONDS[attempt - 1]) + continue + last_exc = None + break + + if last_exc is not None or runtimes is None: + # R8: every success path below ends in `notify_all()` — this give-up path must too, + # or a `lease()` blocked in `_wait_bounded(poll=False)` (the `abandon is None` case) + # waits forever for a reconcile that already gave up. + async with self._state_lock: + self._state_lock.notify_all() + return # stays `_down`; the next `mark_unhealthy` (or a lease-triggered wait) retries. + + # M12: recovery is judged by re-probing `healthy()` (M2's port), never by reading `state` + # back — the scheduler is what wrote `state` when it demoted this world, so trusting it + # here would be reading our own signal as independent proof. R13 (spine v1.12 §4.5b): + # `healthy` now rides the port's non-reentrancy rule, so these probes go under + # `_provider_lock` too. + healthy_by_index: dict[int, bool] = {} + async with self._provider_lock: + for runtime in runtimes: + try: + healthy_by_index[runtime.world_index] = await self._provisioner.healthy( + runtime, work_directory=self._work_directory + ) + except Exception: # noqa: BLE001 + healthy_by_index[runtime.world_index] = False + + achieved = {runtime.world_index for runtime in runtimes} + async with self._state_lock: + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + if healthy_by_index.get(runtime.world_index, False): + was_down = runtime.world_index in self._down + self._down.discard(runtime.world_index) + if runtime.world_index not in self._leased: + self._available.add(runtime.world_index) + if was_down and runtime.state is RuntimeState.READY: + self._fresh.add(runtime.world_index) # m9 + # `provision` reconciles to exactly `instances` worlds (a conformance-gate degrade can + # shrink `achieved` below what this pool started with) — anything no longer returned + # is gone, not merely unhealthy. + for stale in [index for index in self._runtimes if index not in achieved]: + self._runtimes.pop(stale, None) + self._available.discard(stale) + self._down.discard(stale) + self._fresh.discard(stale) + # m3: NOT `_leased.discard(stale)` — an in-flight scenario may still hold this + # index's lease (e.g. a conformance degrade shrinking `achieved` mid-scenario); + # dropping the lease record here would make its later `release()`/ + # `mark_unhealthy()` a silent no-op. Those methods already guard on + # `world_index in self._runtimes`, so leaving `_leased` alone and letting them + # reconcile it lazily is correct. + self._state_lock.notify_all() + + async def close(self) -> None: + async with self._state_lock: + if self._closed: + return # R5: idempotent, matching spine §4 point 4 ("close is idempotent"). + self._closed = True + self._closing = True + task = self._reconcile_task + # R5: wake anything blocked in `lease()`'s `_wait_bounded(poll=False)` so it re-checks + # `_closed` instead of waiting for a recovery that will never come. + self._state_lock.notify_all() + + if task is not None: + # R4: `ProcessRuntimeProvider.provision`/`reset`/`healthy` are `asyncio.to_thread` — + # cancelling the awaiting coroutine does NOT stop the underlying thread, so + # cancelling immediately just races the hard-clean below against a `provision()` + # still repopulating `self._runtimes`/the worlds directory. Wait for the real work to + # finish on its own first; only cancel (accepting the thread may still leak, same + # bounded tradeoff as an abandoned scenario phase) if it blows the bound. + done, pending = await asyncio.wait({task}, timeout=_CLOSE_RECONCILE_WAIT_SECONDS) + for pending_task in pending: + pending_task.cancel() + await asyncio.gather(*done, *pending, return_exceptions=True) + + async with self._provider_lock: + await self._provisioner.close(work_directory=self._work_directory) + + async def _log(self, message: str, *, level: str = "error") -> None: + if self._outbound is None: + return + try: + # R9: reuses `world_unhealthy.cause`'s own sanitizer — a `provision()` failure + # routinely carries a postgres error string with the DSN, and outbound-channels.md + # requires redaction (no endpoint userinfo) before anything crosses the wire. + await self._outbound.log(level=level, message=_sanitize_cause(message)) + except Exception: # noqa: BLE001 - B3: outbound failures are never fatal. + pass + + +# --- the scenario loop -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RunResult: + receipts: tuple[ResultReceipt, ...] + aborted: ReceiptFailure | None + + +def _skipped_receipt(scenario: Scenario) -> ResultReceipt: + # Exact body per outbound-channels.md Channel 2, "skipped receipt body (exact)". + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=1, + world_index=None, + status="skipped", + sub_goals=(), + evaluations=(), + call=None, + failure=None, + ) + + +def _unjudged(sub_goals: Sequence[SubGoal]) -> tuple[SubGoalResult, ...]: + return tuple( + # R11: outbound-channels.md pins `judged` as `SubGoal.judged != ""`, not `bool(...)` — + # they agree for every `str` but `bool` is not what the contract names. + SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "") + for goal in sub_goals + ) + + +_LEAK_HEADROOM = 10 # R1: spine §1's hosted `scenario_count` admission range is 1..10 -- the most +# phase threads that can ever be simultaneously abandoned (leaked) in one job. + + +@dataclass +class _ScenarioContext: + """R7: `_run_scenario` records the world/attempt it is currently working on here as it goes, + so a crash that escapes every handled path still lets `worker()` report the REAL + world_index/scenario_attempt on the `driver_crashed` receipt instead of always None/1.""" + + world_index: int | None = None + attempt: int = 1 + + +@dataclass(frozen=True) +class _PendingRetryReceipt: + """R3: carries attempt-1's already-built `_Retry` outcome across the retry-lease boundary, so + a cancel/abort landing anywhere between "attempt 1 finished" and "attempt 2 actually starts" + still reports what attempt 1 produced instead of losing it to skipped-synthesis (the same + defect M8 fixed on the `NoWorldsAvailable` branch, on the other post-attempt-1 exit).""" + + world_index: int + attempt: int + outcome: "_Retry" + + +class HostedScheduler: + """Drains a job's scenario list across a `WorldPool`, one asyncio task per scenario — lease() + blocking when the pool is saturated is what caps concurrency at W, so nothing here re-derives + a worker count. Retry is fixed at one extra attempt on a fresh world (spine §5 step 4), gated + on `FailureDomain` per the P9 brief: retryable domains retry once, deterministic ones do not. + """ + + def __init__( + self, + *, + pool: WorldPool, + world_factory: WorldFactory, + call_runner: CallRunner, + outbound: OutboundPort, + job_seed: int, + cancel_requested: Callable[[], bool] | None = None, + ) -> None: + self._pool = pool + self._world_factory = world_factory + self._call_runner = call_runner + self._outbound = outbound + self._job_seed = job_seed + self._cancel_requested = cancel_requested or (lambda: False) + self._executor: ThreadPoolExecutor | None = None + + async def run(self, scenarios: Sequence[Scenario]) -> RunResult: + results: list[ResultReceipt | None] = [None] * len(scenarios) + abort_holder: list[ReceiptFailure | None] = [None] + + # R1: a dedicated executor for scenario phase threads — never the loop's default + # executor, which the provider's own `to_thread` calls (process_runtime.py) also use, and + # whose capacity a leaked phase thread would starve globally. One worker per live world + # plus headroom for the worst case of every admitted scenario leaking its own abandoned + # thread at once (world-handle-interface.md: "its thread leaks, bounded by scenario + # count"). + self._executor = ThreadPoolExecutor( + max_workers=self._pool.effective_size + _LEAK_HEADROOM, thread_name_prefix="hosted-scenario" + ) + try: + + async def worker(index: int, scenario: Scenario) -> None: + if abort_holder[0] is not None or self._cancel_requested(): + return + context = _ScenarioContext() + try: + results[index] = await self._run_scenario( + scenario, index, abort_holder=abort_holder, context=context + ) + except NoWorldsAvailable as exc: + abort_holder[0] = _failure("world_pool_exhausted", str(exc)) + except asyncio.CancelledError: + raise + except BaseException as exc: # noqa: BLE001 + # B3: the scheduler's own machinery crashing must not suppress every other + # scenario's receipt — `gather(return_exceptions=True)` below is the second + # half of that guarantee. + results[index] = await self._driver_crashed_receipt( + scenario, exc, world_index=context.world_index, scenario_attempt=context.attempt + ) + + tasks = [asyncio.create_task(worker(i, s)) for i, s in enumerate(scenarios)] + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + receipts: list[ResultReceipt] = [] + for index, scenario in enumerate(scenarios): + receipt = results[index] + if receipt is None: + receipt = _skipped_receipt(scenario) + await self._emit(self._outbound.receipt(receipt), what="receipt") + receipts.append(receipt) + return RunResult(receipts=tuple(receipts), aborted=abort_holder[0]) + finally: + # R1: never block `run()` on abandoned threads — `shutdown(wait=True)` would hang + # this coroutine exactly like the bug this fixes. Queued-but-unstarted work is + # cancelled; already-running (leaked) threads are the contract's own accepted, + # bounded tradeoff (world-handle-interface.md's "the job TTL is the backstop"). + self._executor.shutdown(wait=False, cancel_futures=True) + + async def _emit(self, awaitable: Awaitable[None], *, what: str) -> None: + # B3: `OutboundPort` exceptions are best-effort telemetry — never receipt-affecting and + # never fatal to the run. Logged through the same port when logging itself doesn't also + # fail; swallowed otherwise rather than let a transport hiccup kill the scenario loop. + try: + await awaitable + except Exception as exc: # noqa: BLE001 + try: + await self._outbound.log(level="error", message=f"outbound.{what} failed: {exc}") + except Exception: # noqa: BLE001 + pass + + async def _driver_crashed_receipt( + self, scenario: Scenario, exc: BaseException, *, world_index: int | None, scenario_attempt: int + ) -> ResultReceipt: + failure = _failure("driver_crashed", f"{type(exc).__name__}: {exc}") + try: + # R7: best-effort — every declared goal, `held: null`, matching the errored-receipt + # body's rule. Falls back to `()` only when reading `sub_goals` itself is what crashed + # (the one case with no goal list to report at all). + sub_goals = _unjudged(scenario.sub_goals) + except Exception: # noqa: BLE001 + sub_goals = () + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=scenario_attempt, + world_index=world_index, + status="errored", + sub_goals=sub_goals, + evaluations=(), + call=None, + failure=failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + async def _emit_pending_retry_receipt( + self, scenario: Scenario, pending: "_PendingRetryReceipt" + ) -> ResultReceipt: + # R3: the single shape both post-attempt-1 "never got to run attempt 2" exits emit. + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=pending.attempt, + world_index=pending.world_index, + status="errored", + sub_goals=pending.outcome.sub_goals, + evaluations=(), + call=pending.outcome.call, + failure=pending.outcome.failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + async def _lease_or_abandon( + self, *, exclude: frozenset[int], abort_holder: list[ReceiptFailure | None] + ) -> tuple[int, EnvironmentRuntime] | None: + def _abandon() -> bool: + return abort_holder[0] is not None or self._cancel_requested() + + return await self._pool.lease(exclude=exclude, abandon=_abandon) + + async def _run_scenario( + self, + scenario: Scenario, + scenario_index: int, + *, + attempt: int = 1, + tried: frozenset[int] = frozenset(), + pre_leased: tuple[int, EnvironmentRuntime] | None = None, + pending_retry: "_PendingRetryReceipt | None" = None, + abort_holder: list[ReceiptFailure | None], + context: "_ScenarioContext", + ) -> ResultReceipt | None: + if pre_leased is not None: + world_index, runtime = pre_leased + else: + leased = await self._lease_or_abandon(exclude=tried, abort_holder=abort_holder) + if leased is None: + return None # B5: cancelled/aborted while queued — never got a world + world_index, runtime = leased + + context.world_index = world_index # R7: the real values for a driver_crashed receipt + context.attempt = attempt + + # B5: re-check immediately after `lease()` returns — a cancel/abort landing while this + # worker was queued must not let a freshly granted world start work it can never finish + # inside the flush window. + if abort_holder[0] is not None or self._cancel_requested(): + await self._pool.release(world_index) + if pending_retry is not None: + # R3: attempt 1 already ran on `pending_retry.world_index` and produced a real + # outcome — this is the retry continuation (this world was never used for it). + return await self._emit_pending_retry_receipt(scenario, pending_retry) + return None + + world_resolved = False # B3: the leased world must be released/discarded exactly once + try: + await self._emit( + self._outbound.scenario_started( + scenario_key=scenario.scenario_key, + world_index=world_index, + scenario_attempt=attempt, + ), + what="scenario_started", + ) + + rng = random.Random(self._job_seed + scenario_index) + outcome: ResultReceipt | _Retry + try: + world = await self._world_factory.create(runtime, rng=rng) + except Exception as exc: # noqa: BLE001 + # B3: `world_factory.create()` failing (e.g. a PostgresStore connect failure) is + # the same shape as a mid-scenario `WorldUnavailable` — the world is unusable, not + # the scenario code. Deliberately narrow to just this call: `_execute()` has its + # own exhaustive internal exception handling (`_run_phase`/`_invoke`), so anything + # that still escapes it is a genuine scheduler bug and belongs in `driver_crashed` + # (via `worker()`'s `BaseException` catch), not swallowed into `world_unavailable`. + outcome = _Retry( + _failure("world_unavailable", f"{type(exc).__name__}: {exc}"), + sub_goals=_unjudged(scenario.sub_goals), + call=None, + mark_unhealthy=True, + ) + else: + outcome = await self._execute(scenario, world, runtime, world_index, attempt=attempt) + + if isinstance(outcome, _Retry): + # R6: `mark_unhealthy()` itself emits `world_unhealthy` now (every demotion path + # goes through it) — no separate emit needed here. + if outcome.mark_unhealthy: + await self._pool.mark_unhealthy(world_index, cause=outcome.failure.message) + else: + await self._pool.release(world_index) + world_resolved = True + + if attempt >= 2: + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status="errored", + sub_goals=outcome.sub_goals, + evaluations=(), + call=outcome.call, + failure=outcome.failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + # R3: attempt 1's outcome, carried forward so either exit below that never gets to + # start attempt 2 can still report it instead of losing it to skipped-synthesis. + pending = _PendingRetryReceipt(world_index=world_index, attempt=attempt, outcome=outcome) + try: + next_leased = await self._lease_or_abandon( + exclude=tried | {world_index}, abort_holder=abort_holder + ) + except NoWorldsAvailable as exc: + # M8: this scenario already ran and produced a real attempt-1 failure — losing + # it to skipped-synthesis just because the retry lease found nothing would + # report "never ran" for a scenario that manifestly did. + abort_holder[0] = _failure("world_pool_exhausted", str(exc)) + return await self._emit_pending_retry_receipt(scenario, pending) + + if next_leased is None: + # R3: same defect as the branch above, reached via cancel/abort instead of + # pool exhaustion. + return await self._emit_pending_retry_receipt(scenario, pending) + next_index, next_runtime = next_leased + await self._emit( + self._outbound.scenario_retried( + scenario_key=scenario.scenario_key, from_world=world_index, to_world=next_index + ), + what="scenario_retried", + ) + return await self._run_scenario( + scenario, + scenario_index, + attempt=2, + tried=tried | {world_index}, + pre_leased=(next_index, next_runtime), + pending_retry=pending, + abort_holder=abort_holder, + context=context, + ) + + # M13: a plain terminal receipt is either a real passed/failed verdict (release — the + # world is fine) or a non-retryable fault from `_fault()`. For the latter, an + # exception/overrun code means the world is half-applied and must be discarded rather + # than handed to the next scenario; `ready_not_ready` is a clean verdict and keeps + # `release()`. + if outcome.failure is not None and outcome.failure.code in _DISCARD_ON_ERROR_CODES: + await self._pool.mark_unhealthy(world_index, cause=outcome.failure.message) + else: + await self._pool.release(world_index) + world_resolved = True + await self._emit(self._outbound.receipt(outcome), what="receipt") + return outcome + finally: + if not world_resolved: + # B3: something blew past every handled path above (a bug in this module itself) + # — the world must not be silently stranded outside the pool's bookkeeping. + # Discarded rather than released: an exception here leaves its state unknown, and + # world-handle-interface.md's own exception rule is "discarded and re-provisioned, + # never reused." + await self._pool.mark_unhealthy( + world_index, cause="scenario driver crashed while holding this world" + ) + + async def _execute( + self, scenario: Scenario, world: World, runtime: EnvironmentRuntime, world_index: int, *, attempt: int + ) -> "ResultReceipt | _Retry": + setup = await _run_phase( + scenario.setup, world, timeout=SETUP_TIMEOUT_SECONDS, phase="setup", executor=self._executor + ) + if setup.failure is not None: + return self._fault(scenario, world_index, attempt, setup.failure, sub_goals=_unjudged(scenario.sub_goals)) + + read_only = world.read_only() + ready = await _run_phase( + scenario.ready, read_only, timeout=READY_TIMEOUT_SECONDS, phase="ready", executor=self._executor + ) + if ready.failure is not None: + return self._fault(scenario, world_index, attempt, ready.failure, sub_goals=_unjudged(scenario.sub_goals)) + verdict = _classify_ready(ready.value) + if verdict.broken: + return self._fault( + scenario, world_index, attempt, _failure("ready_broken", f"ready() returned {ready.value!r}"), + sub_goals=_unjudged(scenario.sub_goals), + ) + if not verdict.held: + return self._fault( + scenario, world_index, attempt, _failure("ready_not_ready", verdict.reason or ""), + sub_goals=_unjudged(scenario.sub_goals), + ) + + try: + call_outcome = await self._call_runner.run(scenario, runtime) + except WorldUnavailable as exc: + return _Retry(_failure("world_unavailable", str(exc)), sub_goals=_unjudged(scenario.sub_goals), call=None, mark_unhealthy=True) + except CallAborted as exc: + call = self._call_summary(exc.partial) + return self._fault( + scenario, world_index, attempt, _failure("call_failed", str(exc)), + sub_goals=_unjudged(scenario.sub_goals), call=call, + ) + except Exception as exc: # noqa: BLE001 + # B3: the call runner crashing outright (not a `CallAborted` it chose to raise) is the + # same world-handle-interface.md v3.3 row — "the simulated-call machinery crashed" — + # just with no partial evidence to report. + return self._fault( + scenario, world_index, attempt, _failure("call_failed", f"{type(exc).__name__}: {exc}"), + sub_goals=_unjudged(scenario.sub_goals), call=None, + ) + + calls = list(call_outcome.calls) # m12: `folder.py::_RUNNABLE` expects a list, not a tuple. + if not calls: + # M10: unconditioned on `turns` — an empty list must never reach checks regardless of + # whether the simulator observed a turn (world-handle-interface.md "Coverage + # guarantee": "An empty list is never handed to checks"). + failure = _failure("evidence_missing", "no tool calls were captured for this scenario's call") + return _Retry(failure, sub_goals=_unjudged(scenario.sub_goals), call=self._call_summary(call_outcome), mark_unhealthy=False) + + if not scenario.sub_goals: + # m7: `all(())` is vacuously True — a scenario declaring zero sub-goals must not read + # as a silent pass. + return self._fault( + scenario, world_index, attempt, + _failure("check_broken", "scenario declared zero sub_goals — a vacuous pass is forbidden"), + sub_goals=(), call=self._call_summary(call_outcome), + ) + + sub_goal_results: list[SubGoalResult] = [] + check_handle = world.read_only() + broken_failure: ReceiptFailure | None = None + for goal in scenario.sub_goals: + if broken_failure is not None: + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + outcome = await _run_phase( + goal.check, check_handle, calls, timeout=CHECK_TIMEOUT_SECONDS, phase="check", executor=self._executor + ) + if outcome.failure is not None: + if outcome.failure.code == "world_unavailable": + return _Retry( + outcome.failure, sub_goals=tuple(sub_goal_results) + _unjudged([goal]) + _unjudged(scenario.sub_goals[len(sub_goal_results) + 1 :]), + call=self._call_summary(call_outcome), mark_unhealthy=True, + ) + broken_failure = outcome.failure + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + verdict = _classify_check(outcome.value) + if verdict.broken: + broken_failure = _failure("check_broken", f"{goal.name}: check() returned {outcome.value!r}") + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + sub_goal_results.append(SubGoalResult(name=goal.name, held=verdict.held, reason=verdict.reason, judged=goal.judged != "")) + + if broken_failure is not None: + return self._fault( + scenario, world_index, attempt, broken_failure, sub_goals=tuple(sub_goal_results), call=self._call_summary(call_outcome), + ) + + status = "passed" if all(result.held for result in sub_goal_results) else "failed" + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status=status, + sub_goals=tuple(sub_goal_results), + evaluations=(), + call=self._call_summary(call_outcome), + failure=None, + ) + + @staticmethod + def _call_summary(outcome: CallOutcome | None) -> CallSummary | None: + if outcome is None: + return None + return CallSummary( + started_at=outcome.started_at, + ended_at=outcome.ended_at, + duration_ms=outcome.duration_ms, + turns=outcome.turns, + transcript_artifact=outcome.transcript_artifact, + recording_artifacts=outcome.recording_artifacts, + ) + + def _fault( + self, + scenario: Scenario, + world_index: int, + attempt: int, + failure: ReceiptFailure, + *, + sub_goals: tuple[SubGoalResult, ...], + call: CallSummary | None = None, + retry: bool | None = None, + ) -> "ResultReceipt | _Retry": + should_retry = _is_retryable(failure.code) if retry is None else retry + if should_retry: + return _Retry(failure, sub_goals=sub_goals, call=call, mark_unhealthy=failure.code == "world_unavailable") + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status="errored", + sub_goals=sub_goals, + evaluations=(), + call=call, + failure=failure, + ) + + +@dataclass(frozen=True) +class _Retry: + failure: ReceiptFailure + sub_goals: tuple[SubGoalResult, ...] + call: CallSummary | None + mark_unhealthy: bool + + +__all__ = [ + "CHECK_TIMEOUT_SECONDS", + "READY_TIMEOUT_SECONDS", + "SETUP_TIMEOUT_SECONDS", + "Call", + "CallAborted", + "CallOutcome", + "CallRunner", + "CallSummary", + "Evaluation", + "HostedScheduler", + "NoWorldsAvailable", + "OutboundPort", + "ReadOnlyWorld", + "ReceiptFailure", + "ResultReceipt", + "RunResult", + "Scenario", + "SubGoal", + "SubGoalResult", + "World", + "WorldFactory", + "WorldPool", + "WorldProvisioner", +] diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py new file mode 100644 index 00000000..d5ede280 --- /dev/null +++ b/tests/harness/test_hosted_scheduler.py @@ -0,0 +1,1555 @@ +"""`hosted_scheduler.py` against in-memory fakes — no real provisioner, no real postgres. + +`asyncio.run` drives every `async def` seam here, matching `test_process_runtime.py`'s own +convention (no pytest-asyncio dependency in this repo). +""" + +from __future__ import annotations + +import asyncio +import contextlib +import random +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from fi.alk.harness import hosted_scheduler as hs +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState +from fi.alk.harness.world.errors import ( + WorldReadOnly, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) + +# --- fakes --------------------------------------------------------------------------------- + + +def _runtime(index: int, state: RuntimeState = RuntimeState.READY) -> EnvironmentRuntime: + return EnvironmentRuntime( + runtime_id=f"digest:w{index}", world_index=index, bundle_digest="digest", state=state + ) + + +class FakeProvisioner: + """Mirrors `ProcessRuntimeProvider`'s real async shape, including the two properties the + review named as fidelity gaps: `reset_scripts` lets a test script one world's next N reset + outcomes (anything unscripted resets clean to READY), and every provider call — including + `healthy()` (R13: spine v1.12 §4.5b folded it into the same non-reentrant set) — goes through + `_serialized`, which both yields (`await asyncio.sleep(0)` — so a genuine overlap has a real + chance to interleave) and asserts no second call is ever in flight at the same time, matching + "not reentrant" (B1/B2's own regression test).""" + + def __init__( + self, instances: int, *, reset_scripts: dict[int, list[RuntimeState]] | None = None + ) -> None: + self.instances = instances + self.reset_scripts = reset_scripts or {} + self.provision_calls = 0 + self.reset_calls = 0 + self.closed = False + self._runtimes = {i: _runtime(i) for i in range(instances)} + self._busy = False + + @contextlib.asynccontextmanager + async def _serialized(self): + # The `try/finally` wraps the yielding `sleep(0)` too — `close()` cancelling an in-flight + # reconcile (M6) must still clear `_busy`, or a cancellation lands this assertion stuck + # True forever and fails every later call in the same test for the wrong reason. + assert not self._busy, "provider port called reentrantly (provision/reset/healthy/close overlap)" + self._busy = True + try: + await asyncio.sleep(0) + yield + finally: + self._busy = False + + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + async with self._serialized(): + self.provision_calls += 1 + for index in range(instances): + if index not in self._runtimes or self._runtimes[index].state in ( + RuntimeState.STOPPED, RuntimeState.UNHEALTHY, + ): + self._runtimes[index] = _runtime(index, RuntimeState.READY) + return [self._runtimes[index] for index in range(instances) if index in self._runtimes] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(): + self.reset_calls += 1 + script = self.reset_scripts.get(runtime.world_index) + runtime.state = script.pop(0) if script else RuntimeState.READY + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(): + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory: Path) -> None: + async with self._serialized(): + self.closed = True + + +class InMemoryWorld: + """The six-verb surface, backed by a plain dict instead of postgres — enough to exercise the + scheduler's own control flow without a `PostgresStore`.""" + + def __init__(self, world_index: int, rng: random.Random, *, read_only: bool = False) -> None: + self.world_index = world_index + self.rng = rng + self._read_only = read_only + self.rows: dict[str, list[dict[str, Any]]] = {} + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return dict(self.rows) if table is None else {table: list(self.rows.get(table, []))} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + if self._read_only: + raise WorldReadOnly("read-only world") + self.rows.setdefault(collection, []).append(record) + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + if self._read_only: + raise WorldReadOnly("read-only world") + return 0 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + if self._read_only: + raise WorldReadOnly("read-only world") + return 0 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> hs.Call: + if self._read_only: + raise WorldReadOnly("read-only world") + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + return [] + + def read_only(self) -> "InMemoryWorld": + return InMemoryWorld(self.world_index, self.rng, read_only=True) + + +class FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> InMemoryWorld: + return InMemoryWorld(runtime.world_index, rng) + + +@dataclass +class FakeSubGoal: + name: str + fn: Any + judged: str = "" + + def check(self, world: Any, calls: Any) -> object: + return self.fn(world, calls) + + +@dataclass +class FakeScenario: + scenario_key: str + scenario_id: str + sub_goals: list[FakeSubGoal] = field(default_factory=list) + setup_fn: Any = lambda world: None + ready_fn: Any = lambda world: None + + def setup(self, world: Any) -> object: + return self.setup_fn(world) + + def ready(self, world: Any) -> object: + return self.ready_fn(world) + + +class FakeCallRunner: + def __init__(self, outcomes: dict[str, Any]) -> None: + self.outcomes = outcomes + self.calls: list[tuple[str, int]] = [] + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + self.calls.append((scenario.scenario_key, runtime.world_index)) + outcome = self.outcomes[scenario.scenario_key] + if isinstance(outcome, Exception): + raise outcome + return outcome + + +class FakeOutbound: + def __init__(self) -> None: + self.events: list[tuple[str, dict[str, Any]]] = [] + self.receipts: list[hs.ResultReceipt] = [] + + async def scenario_started(self, *, scenario_key: str, world_index: int, scenario_attempt: int) -> None: + self.events.append(("scenario_started", {"scenario_key": scenario_key, "world_index": world_index, "scenario_attempt": scenario_attempt})) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + self.events.append(("scenario_retried", {"scenario_key": scenario_key, "from_world": from_world, "to_world": to_world})) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + self.events.append(("world_unhealthy", {"world_index": world_index, "cause": cause})) + + async def log(self, *, level: str, message: str) -> None: + self.events.append(("log", {"level": level, "message": message})) + + async def receipt(self, receipt: hs.ResultReceipt) -> None: + self.receipts.append(receipt) + + +class FailingOutbound(FakeOutbound): + """B3 regression fixture: a subset of methods raise, standing in for a dead transport.""" + + def __init__(self, *, fail_on: set[str]) -> None: + super().__init__() + self._fail_on = fail_on + + async def scenario_started(self, **kwargs: Any) -> None: + if "scenario_started" in self._fail_on: + raise ConnectionError("outbound transport down") + await super().scenario_started(**kwargs) + + async def receipt(self, receipt: hs.ResultReceipt) -> None: + if "receipt" in self._fail_on: + raise ConnectionError("outbound transport down") + await super().receipt(receipt) + + +def _call_outcome(turns: int = 1, calls: tuple[hs.Call, ...] = ()) -> hs.CallOutcome: + return hs.CallOutcome( + calls=calls, turns=turns, started_at="2026-08-25T00:00:00.000Z", + ended_at="2026-08-25T00:00:05.000Z", duration_ms=5000, + ) + + +def _pool( + instances: int, *, provisioner: Any | None = None, + reset_scripts: dict[int, list[RuntimeState]] | None = None, + outbound: Any | None = None, +) -> tuple[hs.WorldPool, FakeProvisioner]: + fake = provisioner or FakeProvisioner(instances, reset_scripts=reset_scripts) + pool = hs.WorldPool( + fake, bundle=object(), source=Path("/work/source"), bundle_dir=Path("/work/bundle"), + work_directory=Path("/work"), instances=instances, outbound=outbound, + ) + return pool, fake + + +# --- WorldPool ------------------------------------------------------------------------------- + + +def test_lease_skips_reset_for_a_freshly_provisioned_world_but_not_the_next_lease() -> None: + # m9: a world just handed back by `provision()` is already at the sealed baseline — the + # first lease must not pay for a redundant reset, but a world that has already been used + # once resets normally on its next lease. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + world_index, runtime = await pool.lease() + assert world_index == 0 + assert runtime.state is RuntimeState.READY + assert provisioner.reset_calls == 0 + await pool.release(world_index) + world_index, runtime = await pool.lease() + assert provisioner.reset_calls == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_world_left_unhealthy_by_reset_is_not_handed_out() -> None: + async def scenario() -> None: + pool, provisioner = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}) + await pool.start() + first, _ = await pool.lease() + assert first == 0 # freshly provisioned -- m9 skips this lease's reset + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's (now real) reset hit the scripted UNHEALTHY outcome + await pool.close() + + asyncio.run(scenario()) + + +def test_a_freshly_provisioned_world_failing_its_health_probe_is_not_handed_out() -> None: + # M2: `healthy()` is called unconditionally after reset — including on the m9 fast path, + # which only skips the (expensive) reset call, never the readiness check. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(): + return runtime.world_index != 0 + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + world_index, _ = await pool.lease() + assert world_index == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_an_unhealthy_world_is_reconciled_back_in_the_background() -> None: + async def scenario() -> None: + pool, provisioner = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}) + await pool.start() + first, _ = await pool.lease() + assert first == 0 + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's reset failed and it fell out of rotation + await asyncio.sleep(0.2) + assert provisioner.provision_calls >= 2 # start() + the background reconcile + # T1: the world must be genuinely leasable again, not merely still counted in pool.size. + recovered_index, recovered_runtime = await pool.lease() + assert recovered_index == 0 + assert recovered_runtime.state is RuntimeState.READY + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_excluding_the_only_world_raises_rather_than_hanging() -> None: + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + world_index, _ = await pool.lease() + await pool.release(world_index) + try: + await asyncio.wait_for(pool.lease(exclude=frozenset({world_index})), timeout=1.0) + except hs.NoWorldsAvailable: + pass + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_can_drop_a_world_that_never_recovers() -> None: + async def scenario() -> None: + calls = {"n": 0} + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return [_runtime(0, RuntimeState.READY), _runtime(1, RuntimeState.READY)] + return [_runtime(1, RuntimeState.READY)] # world 0 degraded away, every time + + async def reset(self, runtime, *, work_directory): + runtime.state = RuntimeState.UNHEALTHY if runtime.world_index == 0 else RuntimeState.READY + + async def healthy(self, runtime, *, work_directory): + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(2, provisioner=Provisioner()) + await pool.start() + first, _ = await pool.lease() + assert first == 0 # freshly provisioned -- m9 skips this lease's reset + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's (now real) reset marked it unhealthy + await asyncio.sleep(0.2) + assert pool.size == 1 # the reconcile's own `provision()` never brings world 0 back + await pool.close() + + asyncio.run(scenario()) + + +def test_concurrent_mark_unhealthy_never_calls_provision_reentrantly() -> None: + # T7/B1: two worlds going bad in the same tick must serialize onto one provider call at a + # time — `FakeProvisioner._serialized`'s own assertion is what actually catches a regression. + async def scenario() -> None: + pool, provisioner = _pool(2) + await pool.start() + w0, _ = await pool.lease() + w1, _ = await pool.lease() + await asyncio.gather( + pool.mark_unhealthy(w0, cause="boom0"), pool.mark_unhealthy(w1, cause="boom1") + ) + await asyncio.sleep(0.1) + assert pool.size == 2 + world_index, runtime = await pool.lease() + assert runtime.state is RuntimeState.READY + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_reset_and_a_background_reconcile_never_overlap_on_the_provider() -> None: + # TH-4/B2: the overlap that actually matters is a lease()'s reset() running concurrently with + # a DIFFERENT world's reconcile provision() -- the old coalescer test never drove two + # DIFFERENT provider calls at once; `FakeProvisioner._serialized`'s reentrancy assertion is + # what would catch a `_provider_lock` regression, so this drives it for real. + async def scenario() -> None: + pool, provisioner = _pool(2) + await pool.start() + + # Use up world 1's "fresh" fast path so its NEXT lease pays for a real reset() call -- + # otherwise it would never touch the provider at all and the overlap wouldn't be real. + w1, _ = await pool.lease(exclude=frozenset({0})) + assert w1 == 1 + await pool.release(1) + + w0, _ = await pool.lease(exclude=frozenset({1})) + assert w0 == 0 + + results = await asyncio.gather( + pool.mark_unhealthy(0, cause="boom"), # schedules a background reconcile provision() + pool.lease(exclude=frozenset({0})), # world 1's reset()+healthy() run concurrently + ) + leased = results[1] + assert leased is not None and leased[0] == 1 + await asyncio.sleep(0.1) # let the reconcile finish + assert provisioner.provision_calls >= 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_close_waits_for_an_in_flight_reconcile_before_closing_the_provider() -> None: + # TH-3/R4: `ProcessRuntimeProvider.provision` is `asyncio.to_thread` — cancelling the + # awaiting coroutine does NOT stop the underlying thread. The fake here dispatches through + # `asyncio.to_thread` too (a `threading.Event` gates when the thread-backed work actually + # finishes), so this can only pass if `close()` genuinely waits for the real work instead of + # racing a hard-clean against it. + async def scenario() -> None: + events: list[str] = [] + release_provision = threading.Event() + + def provision_sync(instances: int) -> list[EnvironmentRuntime]: + events.append("provision-start") + release_provision.wait(timeout=5.0) + events.append("provision-end") + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return await asyncio.to_thread(provision_sync, instances) + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory): + events.append("close") + + pool, _ = _pool(1, provisioner=Provisioner()) + release_provision.set() + await pool.start() + release_provision.clear() + events.clear() # drop start()'s own provision-start/-end + + await pool.mark_unhealthy(0, cause="boom") # schedules a reconcile mid-flight + await asyncio.sleep(0.05) # let the reconcile's provision() actually begin on its thread + assert events == ["provision-start"] + + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert events == ["provision-start"], "close() ran the provider's own close() too early" + + release_provision.set() # let the thread-backed provision() finish on its own + await asyncio.wait_for(close_task, timeout=5.0) + assert events == ["provision-start", "provision-end", "close"] + + asyncio.run(scenario()) + + +def test_mark_unhealthy_and_lease_after_close_are_blocked() -> None: + # R5: `close()` latches -- neither a late `mark_unhealthy()` (e.g. a scenario's `finally` + # racing a SIGTERM-triggered close()) nor a fresh `lease()` may touch the provider again once + # the pool has been closed, and a second `close()` is a no-op. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + world_index, _ = await pool.lease() + await pool.close() + assert provisioner.provision_calls == 1 # start()'s call only + assert provisioner.closed is True + + await pool.mark_unhealthy(world_index, cause="late failure") + await asyncio.sleep(0.05) + assert provisioner.provision_calls == 1 # no reconcile spawned post-close + + try: + await asyncio.wait_for(pool.lease(), timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + + await pool.close() # second close() is a no-op + assert provisioner.provision_calls == 1 + + asyncio.run(scenario()) + + +def test_start_degrades_when_provision_returns_fewer_worlds_than_instances() -> None: + # R2: `provision()` legitimately returns fewer worlds than requested (conformance-gate + # failure, `fixed_port`) — spine v1.12 §4: "Fail -> effective parallelism 1 ... + # Loud, never silent," not a `RuntimeError` from this pool. + async def scenario() -> None: + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [_runtime(0)] # only 1 of the 3 requested + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(3, provisioner=Provisioner()) + runtimes = await pool.start() + assert len(runtimes) == 1 + assert pool.effective_size == 1 + world_index, _ = await pool.lease() + assert world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_start_rejects_a_genuinely_malformed_provision_result() -> None: + # R2: the degrade allowance is not a blanket exemption — zero worlds and a non-contiguous + # index set are still rejected as malformed. + class ZeroWorldsProvisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [] + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + class GapProvisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [_runtime(0), _runtime(2)] # world_index 1 missing -- not contiguous + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + async def zero_worlds() -> None: + pool, _ = _pool(2, provisioner=ZeroWorldsProvisioner()) + try: + await pool.start() + except RuntimeError: + pass + else: + raise AssertionError("expected RuntimeError for zero worlds") + + async def gap() -> None: + pool, _ = _pool(3, provisioner=GapProvisioner()) + try: + await pool.start() + except RuntimeError: + pass + else: + raise AssertionError("expected RuntimeError for a non-contiguous index set") + + asyncio.run(zero_worlds()) + asyncio.run(gap()) + + +def test_world_unhealthy_emitted_exactly_once_per_demotion_path() -> None: + # R6: every demotion path now goes through `WorldPool.mark_unhealthy()`, the sole emitter — + # parametrized over three distinct triggers (the fourth, the `_Retry(mark_unhealthy=True)` + # path, is already covered by `test_world_unavailable_retries_once_on_a_fresh_world_and_recovers` + # and `test_world_unhealthy_cause_is_truncated_and_redacted`). + async def run_scheduler_case(setup_fn: Any, sub_goals: Any) -> list[dict[str, Any]]: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1, + ) + scenarios = [FakeScenario("s1", "id-1", setup_fn=setup_fn, sub_goals=sub_goals)] + await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + await pool.close() + return [kwargs for event, kwargs in outbound.events if event == "world_unhealthy"] + + async def m13_discard_branch() -> None: + # the M13 discard branch: a plain `setup_crashed` fault. + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + events = await run_scheduler_case(boom, [FakeSubGoal("g", lambda w, c: None)]) + assert len(events) == 1 + assert events[0]["world_index"] == 0 + + async def b3_finally_branch() -> None: + # the B3 `finally` branch: something blows past every handled path in `_execute`. + class BrokenSubGoals: + def __bool__(self) -> bool: + raise RuntimeError("sub_goals blew up") + + events = await run_scheduler_case(lambda world: None, BrokenSubGoals()) + assert len(events) == 1 + assert events[0]["world_index"] == 0 + + async def lease_reset_failure_demotion() -> None: + # `WorldPool.lease()`'s own reset/health-probe demotion — pool-level, no scheduler. + outbound = FakeOutbound() + pool, _ = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}, outbound=outbound) + await pool.start() + first, _ = await pool.lease() + assert first == 0 + await pool.release(0) + await pool.lease() # world 0's real reset hits the scripted UNHEALTHY outcome + events = [kwargs for event, kwargs in outbound.events if event == "world_unhealthy"] + assert len(events) == 1 + assert events[0]["world_index"] == 0 + await pool.close() + + asyncio.run(m13_discard_branch()) + asyncio.run(b3_finally_branch()) + asyncio.run(lease_reset_failure_demotion()) + + +def test_bundle_dir_is_threaded_through_to_every_provision_call() -> None: + # T7/M1: `bundle_dir` must reach both `start()`'s and `_reconcile()`'s `provision()` calls. + async def scenario() -> None: + seen: list[Path] = [] + + class Provisioner(FakeProvisioner): + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + seen.append(bundle_dir) + return await super().provision( + bundle, source=source, bundle_dir=bundle_dir, work_directory=work_directory, + contract=contract, instances=instances, + ) + + bundle_dir = Path("/work/bundle-xyz") + pool = hs.WorldPool( + Provisioner(1), bundle=object(), source=Path("/work/source"), bundle_dir=bundle_dir, + work_directory=Path("/work"), instances=1, + ) + await pool.start() + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="force a reconcile") + await asyncio.sleep(0.1) + assert seen and all(path == bundle_dir for path in seen) + await pool.close() + + asyncio.run(scenario()) + + +# --- HostedScheduler --------------------------------------------------------------------------- + + +def test_two_scenarios_pass_over_two_worlds() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, provisioner = _pool(2, outbound=outbound) + await pool.start() + call = hs.Call(name="book", arguments={}, ok=True) + runner = FakeCallRunner({"s1": _call_outcome(calls=(call,)), "s2": _call_outcome(calls=(call,))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=100, + ) + scenarios = [ + FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]), + ] + result = await scheduler.run(scenarios) + assert result.aborted is None + assert {r.status for r in result.receipts} == {"passed"} + assert all(r.scenario_attempt == 1 for r in result.receipts) + assert {e for e, _ in outbound.events} == {"scenario_started"} + assert provisioner.provision_calls == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_not_held_check_is_failed_not_errored() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: "the combo was never ordered")])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "failed" + assert receipt.sub_goals[0] == hs.SubGoalResult(name="goal", held=False, reason="the combo was never ordered", judged=False) + assert receipt.failure is None + await pool.close() + + asyncio.run(scenario()) + + +def test_ready_not_ready_errors_without_retrying() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", ready_fn=lambda w: "precondition missing", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure == hs.ReceiptFailure(domain="simulator", stage="running", code="ready_not_ready", message="precondition missing") + assert receipt.scenario_attempt == 1 + assert receipt.sub_goals[0].held is None + assert runner.calls == [] # never reached the call step + # M13: `ready_not_ready` is a clean verdict, not an exception -- the world is released, + # not discarded, so it is immediately leasable again. + world_index, _ = await pool.lease() + assert world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_setup_crash_errors_without_retrying() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=boom, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "setup_crashed" + assert receipt.scenario_attempt == 1 + # M13: an exception outcome discards the world (mark_unhealthy), never release() -- it is + # down, not immediately available, until the background reconcile recovers it. + assert 0 in pool._down + await pool.close() + + asyncio.run(scenario()) + + +def test_world_usage_misuse_in_ready_maps_to_world_usage() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + def misuse(world: Any) -> None: + raise WorldUsageError("cannot invent a table") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", ready_fn=misuse, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "world_usage" + assert receipt.failure.domain == "simulator" + assert 0 in pool._down # M13: discarded, not released + await pool.close() + + asyncio.run(scenario()) + + +def test_state_too_large_from_check_errors_without_retrying() -> None: + # T7: `state_too_large` was never exercised by the original suite. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + + def blow_up(world: Any, calls: Any) -> None: + raise WorldStateTooLarge("table 'events' exceeds the baseline cap") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", blow_up)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "state_too_large" + assert receipt.failure.domain == "simulator" + assert receipt.scenario_attempt == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_phase_over_its_budget_times_out() -> None: + # T4: real scenario code is synchronous — a `time.sleep` body is what actually exercises + # B4's dedicated-executor dispatch. The old `async def` + `asyncio.sleep` version would have + # passed even with B4 unfixed (`wait_for` can always interrupt a coroutine's own suspension + # point), so it never caught the bug. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.05 + try: + def slow(world: Any) -> None: + time.sleep(1.0) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure.code == "setup_timeout" + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await pool.close() + + asyncio.run(scenario()) + + +def test_an_async_phase_over_its_budget_still_times_out() -> None: + # B4: "keep the awaitable branch" — async scenario code (reached via a sync wrapper, as + # `FakeScenario.setup` always is) must still be interruptible on budget. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.05 + try: + async def slow(world: Any) -> None: + await asyncio.sleep(1.0) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure.code == "setup_timeout" + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await pool.close() + + asyncio.run(scenario()) + + +def test_concurrent_worlds_make_progress_while_one_is_blocked_in_sync_code() -> None: + # B4/TH-1: BOTH worlds' setup sleeps -- serialized execution would take >=0.4s, concurrent + # execution ~0.2s. The old version slept only world 0, so serialized and parallel wall times + # were indistinguishable (both ~0.2s) and the elapsed-time assertion alone could not catch a + # regression to serial dispatch; `started_order` now pins the actual interleaving too. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + started_order: list[str] = [] + + def both_slow(world: Any) -> None: + started_order.append(f"start-{world.world_index}") + time.sleep(0.2) + started_order.append(f"end-{world.world_index}") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=both_slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=both_slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + t0 = asyncio.get_running_loop().time() + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + elapsed = asyncio.get_running_loop().time() - t0 + assert all(r.status == "passed" for r in result.receipts) + assert elapsed < 0.3 # serialized would be >=0.4s + # Both worlds must have STARTED before either one ENDED -- genuine concurrency, not just + # a fast total. + assert set(started_order[:2]) == {"start-0", "start-1"}, started_order + await pool.close() + + asyncio.run(scenario()) + + +def test_a_leaked_phase_thread_does_not_starve_a_sibling_world_and_close_still_completes() -> None: + # R1 (highest-ranked missing test): a phase whose thread never returns must not stop the + # NEXT scenario's phase from running, and `pool.close()` must still complete promptly. With + # the shared default executor this used to fail once enough threads leaked; with a dedicated + # `ThreadPoolExecutor` (`shutdown(wait=False, cancel_futures=True)` on `run()` exit) both hold. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.1 + try: + def maybe_runaway(world: Any) -> None: + if world.world_index == 0: + time.sleep(5.0) # abandoned -- this thread never returns + + runner = FakeCallRunner({"s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=maybe_runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=maybe_runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure is not None and result.receipts[0].failure.code == "setup_timeout" + assert result.receipts[1].status == "passed" # world 1's phase ran despite world 0's leak + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await asyncio.wait_for(pool.close(), timeout=2.0) # must not hang behind the leaked thread + + asyncio.run(scenario()) + + +def test_check_broken_leaves_later_subgoals_unjudged() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[ + FakeSubGoal("goal1", lambda w, c: None), + FakeSubGoal("goal2", lambda w, c: 42), # wrong return type -> broken + FakeSubGoal("goal3", lambda w, c: None), + ])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "check_broken" + assert [g.held for g in receipt.sub_goals] == [True, None, None] + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_declared_sub_goals_is_check_broken_not_a_vacuous_pass() -> None: + # m7: `all(())` is vacuously True — must not read as a silent pass. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "check_broken" + assert receipt.sub_goals == () + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unavailable_retries_once_on_a_fresh_world_and_recovers() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, provisioner = _pool(2, outbound=outbound) + await pool.start() + attempts = {"n": 0} + + def check(world: Any, calls: Any) -> None: + attempts["n"] += 1 + if attempts["n"] == 1: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_started") == 2 + assert kinds.count("scenario_retried") == 1 + assert kinds.count("world_unhealthy") == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unhealthy_cause_is_truncated_and_redacted() -> None: + # T7/M7: `cause` is capped at 200 chars and must never carry endpoint credentials. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + leaky_message = "connection failed: postgresql://harness:s3cr3t@localhost:14000/w0 " + ("x" * 300) + + def blow_up(world: Any, calls: Any) -> None: + raise WorldUnavailable(leaky_message) + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", blow_up)])] + await scheduler.run(scenarios) + causes = [kwargs["cause"] for event, kwargs in outbound.events if event == "world_unhealthy"] + assert causes + cause = causes[0] + assert len(cause) <= 200 + assert "s3cr3t" not in cause + assert "://***@" in cause + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unavailable_twice_gives_up_after_the_one_retry() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + def always_fails(world: Any, calls: Any) -> None: + raise WorldUnavailable("always broken") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", always_fails)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.scenario_attempt == 2 + assert receipt.failure.code == "world_unavailable" + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_between_attempt_1_and_attempt_2_reports_errored_not_skipped() -> None: + # R3: attempt 1 ran and produced a real errored outcome; a cancel/abort landing on the + # retry-lease path — the "abandoned while queued for the retry world" site — must not lose + # it to skipped-synthesis. outbound-channels.md defines `skipped` as "never ran," and this + # scenario manifestly did. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + cancel_flag = {"v": False} + + def check(world: Any, calls: Any) -> None: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + + real_lease = pool.lease + lease_calls = {"n": 0} + + async def flaky_lease(*, exclude=frozenset(), abandon=None): + lease_calls["n"] += 1 + if lease_calls["n"] == 2: # the retry-lease call, right after attempt 1 failed + cancel_flag["v"] = True + return await real_lease(exclude=exclude, abandon=abandon) + + pool.lease = flaky_lease # type: ignore[method-assign] + + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "world_unavailable" + assert receipt.scenario_attempt == 1 + assert receipt.world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_during_the_retry_leases_own_health_probe_reports_errored_not_skipped() -> None: + # R3, the SECOND post-attempt-1 `return None` site: a cancel/abort can also land AFTER the + # retry-lease has already granted a world (during ITS OWN reset/healthy await), rather than + # while queued for one -- a distinct code path from the test above, reached at the top of the + # attempt-2 recursive call instead of inside the retry-lease call itself. + async def scenario() -> None: + cancel_flag = {"v": False} + + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + if runtime.world_index == 1: + cancel_flag["v"] = True + async with self._serialized(): + return runtime.state is RuntimeState.READY + + outbound = FakeOutbound() + pool, _ = _pool(2, provisioner=Provisioner(2), outbound=outbound) + await pool.start() + + def check(world: Any, calls: Any) -> None: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "world_unavailable" + assert receipt.scenario_attempt == 1 + assert receipt.world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_evidence_missing_retries_without_marking_the_world_unhealthy() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + seen = {"n": 0} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + seen["n"] += 1 + if seen["n"] == 1: + return _call_outcome(turns=1, calls=()) + return _call_outcome(turns=1, calls=(hs.Call(name="x", arguments={}),)) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + kinds = [event for event, _ in outbound.events] + assert kinds.count("world_unhealthy") == 0 # not a world-health problem + assert kinds.count("scenario_retried") == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_evidence_missing_twice_errors() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + return _call_outcome(turns=1, calls=()) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "evidence_missing" + assert receipt.scenario_attempt == 2 + assert receipt.call is not None # the call step DID run; only evidence capture failed + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_turns_and_zero_calls_is_still_evidence_missing() -> None: + # M10: unconditioned on `turns` — a 0-turn call is an unobserved agent, same as any other + # empty-calls outcome; `evidence_missing` must fire regardless. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + return _call_outcome(turns=0, calls=()) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "evidence_missing" + assert receipt.scenario_attempt == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_aborted_with_partial_evidence_is_reported_not_null() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) # `call_failed` is infrastructure-domain (retryable) — needs a spare world + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + raise hs.CallAborted( + "livekit room dropped", + partial=hs.CallOutcome( + calls=(), turns=0, started_at="2026-08-25T00:00:00.000Z", + ended_at="2026-08-25T00:00:01.000Z", duration_ms=1000, + ), + ) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "call_failed" + assert receipt.call is not None and receipt.call.duration_ms == 1000 + assert receipt.scenario_attempt == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_aborted_with_no_partial_evidence_still_retries() -> None: + # T6/M3: `retry=call is not None` used to suppress the retry whenever `partial` was `None` — + # world-handle-interface.md v3.3 pins `call_failed` as unconditionally retried once. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + def __init__(self) -> None: + self.attempts = 0 + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + self.attempts += 1 + if self.attempts == 1: + raise hs.CallAborted("livekit room never opened", partial=None) + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + runner = Runner() + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + assert runner.attempts == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_runner_raising_a_bare_exception_maps_to_call_failed() -> None: + # T7/B3: a crash from the call runner that is not a `CallAborted` it deliberately raised is + # the same world-handle-interface.md v3.3 row — "the simulated-call machinery crashed". + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": ConnectionError("livekit socket reset")}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "call_failed" + assert receipt.failure.domain == "infrastructure" + assert receipt.scenario_attempt == 2 + assert receipt.call is None + await pool.close() + + asyncio.run(scenario()) + + +def test_a_bug_in_the_driver_itself_becomes_a_driver_crashed_receipt_without_killing_the_run() -> None: + # T7/B3/R7: a crash that blows past every handled path in `_execute` (not the agent, not a + # check, not the call) must land as `driver_crashed` and must not suppress the OTHER + # scenario's receipt. R7: it must also report the REAL world_index the crash happened on + # (world 0, genuinely leased) rather than always `None`. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenSubGoals: + def __bool__(self) -> bool: + raise RuntimeError("sub_goals blew up") + + runner = FakeCallRunner({ + "s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + }) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + broken = FakeScenario("s1", "id-1") + broken.sub_goals = BrokenSubGoals() # type: ignore[assignment] + scenarios = [broken, FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "errored" + assert result.receipts[0].failure.code == "driver_crashed" + assert result.receipts[0].failure.domain == "simulator" + assert result.receipts[0].world_index == 0 # R7: the real world, not always None + assert result.receipts[0].scenario_attempt == 1 + assert result.receipts[1].status == "passed" # the crash did not suppress this receipt + assert len(outbound.receipts) == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_driver_crashed_reports_unjudged_sub_goals_when_they_are_readable() -> None: + # R7: `driver_crashed` must carry every declared sub-goal (`held: null`) when `sub_goals` + # itself is perfectly readable — only a crash reading `sub_goals` falls back to `()`. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenReadOnly: + world_index = 0 + + def read_only(self) -> Any: + raise RuntimeError("world.read_only() blew up") + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return BrokenReadOnly() + + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g1", lambda w, c: None), FakeSubGoal("g2", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.world_index == 0 + assert receipt.scenario_attempt == 1 + assert [g.name for g in receipt.sub_goals] == ["g1", "g2"] + assert all(g.held is None and g.reason is None for g in receipt.sub_goals) + await pool.close() + + asyncio.run(scenario()) + + +def test_exactly_one_receipt_per_scenario_key_even_when_one_scenario_crashes() -> None: + # T7/B3: `gather(return_exceptions=True)` + the try/finally around the leased region must + # never produce zero or duplicate receipts for any scenario. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + runner = FakeCallRunner({"s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=boom, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await scheduler.run(scenarios) + assert [r.status for r in result.receipts] == ["errored", "passed"] + keys = [r.scenario_key for r in outbound.receipts] + assert sorted(keys) == ["s1", "s2"] + assert len(keys) == len(set(keys)) + await pool.close() + + asyncio.run(scenario()) + + +def test_outbound_failures_never_kill_the_run_or_change_the_receipt() -> None: + # B3: a completely broken OutboundPort is best-effort telemetry — never fatal, never + # receipt-affecting. + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + outbound = FailingOutbound(fail_on={"scenario_started", "receipt"}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "passed" + assert result.aborted is None + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_after_the_first_scenario_skips_the_rest() -> None: + # T3/TH-2: the original test's `cancel_requested=lambda: True` was true before `run()` was + # even called, so nothing ever launched and the "in-flight scenario finishes" behavior was + # never exercised. Here the flag flips mid-run, from inside the first scenario's own call + # step. TH-2: re-pins the exact six-field `skipped` receipt body outbound-channels.md calls + # "exact" — the assertion that would have caught R3. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + cancel_flag = {"v": False} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + cancel_flag["v"] = True + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(3)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is None # a cancel is not a job-level failure + assert result.receipts[0].status == "passed" # the in-flight scenario finished + assert result.receipts[1] == hs.ResultReceipt( + scenario_key="s1", scenario_id="id-1", scenario_attempt=1, world_index=None, + status="skipped", sub_goals=(), evaluations=(), call=None, failure=None, + ) + assert result.receipts[2] == hs.ResultReceipt( + scenario_key="s2", scenario_id="id-2", scenario_attempt=1, world_index=None, + status="skipped", sub_goals=(), evaluations=(), call=None, failure=None, + ) + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_ready_worlds_aborts_the_run_and_skips_the_rest() -> None: + # T2: the original test raced two workers that both hit `NoWorldsAvailable` independently at + # t~=0 (W=1, both past the pre-check before either leased) — "the rest gets skipped because + # the job aborted" was never actually exercised. Here W=1, scenario 1 genuinely runs (and + # fails on its only world), the provisioner then goes permanently down, and scenarios 2/3 are + # asserted to have never reached the call step at all. + async def scenario() -> None: + provisioner = FakeProvisioner(1) + calls = {"n": 0} + real_provision = provisioner.provision + + async def flaky_provision(bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return await real_provision( + bundle, source=source, bundle_dir=bundle_dir, work_directory=work_directory, + contract=contract, instances=instances, + ) + raise RuntimeError("provisioner is down") + + provisioner.provision = flaky_provision # type: ignore[method-assign] + outbound = FakeOutbound() + pool, _ = _pool(1, provisioner=provisioner, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + if scenario.scenario_key == "s1": + raise WorldUnavailable("world 0's schema is gone") + raise AssertionError(f"{scenario.scenario_key} should never have reached the call step") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s3", "id-3", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is not None + assert result.aborted.domain == "infrastructure" + assert result.aborted.code == "world_pool_exhausted" + statuses = {r.scenario_key: r.status for r in result.receipts} + assert statuses["s1"] == "errored" # M8: ran and failed -- must not read as "never ran" + assert statuses["s2"] == "skipped" + assert statuses["s3"] == "skipped" + started = [event for event, _ in outbound.events if event == "scenario_started"] + assert len(started) == 1 # s2/s3 were genuinely never launched + + asyncio.run(scenario()) + + +def test_pool_size_caps_concurrent_scenario_execution() -> None: + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + concurrency = {"now": 0, "max": 0} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + concurrency["now"] += 1 + concurrency["max"] = max(concurrency["max"], concurrency["now"]) + await asyncio.sleep(0.02) + concurrency["now"] -= 1 + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + outbound = FakeOutbound() + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]) for i in range(4)] + result = await scheduler.run(scenarios) + assert concurrency["max"] == 1 + assert all(r.status == "passed" for r in result.receipts) + await pool.close() + + asyncio.run(scenario()) + + +def test_scenario_seed_is_job_seed_plus_index() -> None: + # T5: one scenario at index 0 cannot distinguish `+index` from `+0` — two scenarios can. + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + seen_first_draw: dict[str, int] = {} + + def make_setup(key: str): + def setup(world: Any) -> None: + seen_first_draw[key] = world.rng.randint(0, 10**9) + return setup + + outbound = FakeOutbound() + runner = FakeCallRunner({ + "s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + }) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=777) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=make_setup("s1"), sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=make_setup("s2"), sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + await scheduler.run(scenarios) + assert seen_first_draw["s1"] == random.Random(777 + 0).randint(0, 10**9) + assert seen_first_draw["s2"] == random.Random(777 + 1).randint(0, 10**9) + await pool.close() + + asyncio.run(scenario()) + + +def test_a_retry_reseeds_the_rng_identically() -> None: + # world-handle-interface.md Determinism: "a retry re-seeds identically" — attempt 2 must draw + # the SAME first value as attempt 1 did (both `Random(job.seed + scenario_index)`), not + # continue attempt 1's stream and not use a different seed. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + draws: list[int] = [] + attempts = {"n": 0} + + def setup(world: Any) -> None: + draws.append(world.rng.randint(0, 10**9)) + attempts["n"] += 1 + if attempts["n"] == 1: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=555) + scenarios = [FakeScenario("s1", "id-1", setup_fn=setup, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + assert result.receipts[0].status == "passed" + assert result.receipts[0].scenario_attempt == 2 + assert len(draws) == 2 + expected = random.Random(555 + 0).randint(0, 10**9) + assert draws[0] == expected + assert draws[1] == expected + await pool.close() + + asyncio.run(scenario()) From 116b269431b732d1d3d850e066016ad6f0511e5c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 15:21:35 +0530 Subject: [PATCH 08/20] =?UTF-8?q?feat(harness):=20in-sandbox=20provisioner?= =?UTF-8?q?=20=E2=80=94=20seed,=20baseline,=20worlds,=20reset,=20conforman?= =?UTF-8?q?ce?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The provisioner's second half per seam contract v1.12: seed/migration application from the verified bundle under the store's own user, secrets loaded and deleted before any customer process with purposes from job.json, baseline freeze for all three engines (explicit redis SAVE, node-name-free rabbitmq mnesia path), per-world clone and reset with polled readiness promotion, the two-world conformance gate that never raises, the healthy() port method (demote-only), dead shared-engine respawn, reverse-order termination with engine-appropriate signals, and typed §2f failure codes on every path. Plus the P5-round-2 rider cleanups (bundle_v2, process_preflight) and the run_world_check exception-message truncation in checks.py. Five cold review rounds; the round-5 gate closed on mutation evidence. Known defects (rabbitmq datadir_copy baselines are world-0-only — disclosed in-code with the definitions-export follow-up recorded; sentinel read-only guard is defense-in-depth; promote-poll cost) are consolidated in .claude/harness-alk/reports/p6-review-r5.md. Signed-off-by: khushalsonawat --- src/fi/alk/harness/bundle_v2.py | 2 +- src/fi/alk/harness/checks.py | 2 +- src/fi/alk/harness/process_preflight.py | 12 +- src/fi/alk/harness/process_runtime.py | 2773 ++++++++++++++++++++++- tests/harness/test_process_preflight.py | 38 +- tests/harness/test_process_runtime.py | 2454 +++++++++++++++++++- 6 files changed, 5152 insertions(+), 129 deletions(-) diff --git a/src/fi/alk/harness/bundle_v2.py b/src/fi/alk/harness/bundle_v2.py index 19b228d6..9cee66ee 100644 --- a/src/fi/alk/harness/bundle_v2.py +++ b/src/fi/alk/harness/bundle_v2.py @@ -1,5 +1,5 @@ """`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape (`hosted-execution- -seams.md` v1.8). +seams.md` v1.9). v1 (`bundle.py`) describes a `command`-per-service compose world and embeds the repository source. v2 describes `/work/source` as already present and a job that starts plain processes on diff --git a/src/fi/alk/harness/checks.py b/src/fi/alk/harness/checks.py index 71ed82f2..8e4c26e0 100644 --- a/src/fi/alk/harness/checks.py +++ b/src/fi/alk/harness/checks.py @@ -130,7 +130,7 @@ def run_world_check( return Outcome( name, False, - f"the check raised {type(failed).__name__}: {failed}", + f"the check raised {type(failed).__name__}: {str(failed)[:200]}", broken=True, ) return Outcome(name, said is None, "" if said is None else str(said)) diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py index 1e342e4c..ce41aa9f 100644 --- a/src/fi/alk/harness/process_preflight.py +++ b/src/fi/alk/harness/process_preflight.py @@ -1,4 +1,4 @@ -"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.8 — as a single gate the +"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.9 — as a single gate the in-sandbox provisioner runs before starting anything. `bundle_v2.py` validates everything decidable from the manifest's own field values alone; this @@ -52,14 +52,12 @@ class PreflightError(RuntimeError): """A §2e checklist rule rejected the bundle. - ``code`` is one of §2e's failure-code table (v1.8): "contract-rule" codes, each named by a + ``code`` is one of §2e's failure-code table (v1.9): "contract-rule" codes, each named by a numbered checklist item's prose, and "mechanical" codes for plumbing failures the contract describes but does not formalize as a rule (a missing bundle file, an out-of-range - ``parallelism``). Every code this module raises is in that table, with one open exception: - ``fixed_port_reserved`` (F11, p5-round1-review) has no §2e entry yet — the collision it - guards against is real (a `fixed_port` aliasing the provisioner's own port-formula bands) but - the frozen v1.8 table predates the rule; flagged for the owner to add in the next amendment, - not silently worked around. + ``parallelism``). Every code this module raises is in that table — including + ``fixed_port_reserved`` (F11, p5-round1-review; added to the table by v1.9), which guards + against a `fixed_port` aliasing the provisioner's own port-formula bands. """ def __init__(self, code: str, message: str) -> None: diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py index ddaafbad..eb7fc5e7 100644 --- a/src/fi/alk/harness/process_runtime.py +++ b/src/fi/alk/harness/process_runtime.py @@ -1,4 +1,4 @@ -"""The execution half of the in-sandbox provisioner — `hosted-execution-seams.md` v1.8, §2b/§3/§4. +"""The execution half of the in-sandbox provisioner — `hosted-execution-seams.md` v1.12, §2b/§3/§4/§5. Builds one world's running processes from an already-`preflight_bundle`-cleared `EnvironmentBundleV2`: port allocation, `{{...}}` placeholder rendering, copy-based build trees, @@ -25,22 +25,25 @@ from __future__ import annotations +import base64 +import json import logging import os import pwd import re import secrets import shutil +import signal import socket import subprocess import time import urllib.error import urllib.request -from dataclasses import dataclass +from dataclasses import dataclass, replace from enum import Enum from pathlib import Path -from typing import Callable, Protocol, Sequence -from urllib.parse import urlsplit +from typing import Any, Callable, Protocol, Sequence +from urllib.parse import quote as urlquote, urlsplit from pydantic import BaseModel, Field, JsonValue @@ -55,6 +58,7 @@ ReadinessProbeV2, SecretPurpose, SourceProcess, + StoreEntry, ) logger = logging.getLogger(__name__) @@ -711,7 +715,13 @@ def build_process_tree( class SpawnedProcess(Protocol): """What both a real `subprocess.Popen` wrapper and a test fake must provide — enough for - `depends_on`'s `log_marker` variant and for cleanup, nothing engine-specific.""" + `depends_on`'s `log_marker` variant and for cleanup, nothing engine-specific. + + M7, p6-review-r1: `terminate()` alone only SENDS a signal — it does not wait for the process + to actually exit, which several callers used to assume by running a `copytree`/`rmtree`/port- + reuse on the very next line. `wait()`/`kill()` let `_terminate_and_wait` (below) make that + wait real, with a hard escalation for a process that ignores the polite signal. + """ def is_running(self) -> bool: ... @@ -719,6 +729,22 @@ def captured_output(self) -> str: ... def terminate(self) -> None: ... + def interrupt(self) -> None: + """N12, p6-review-r2: SIGINT — postgres's own FAST shutdown (`pg_ctl -m fast`): rolls + back in-flight transactions and disconnects clients immediately, unlike `terminate()`'s + SIGTERM, which postgres treats as a SMART shutdown that waits for every client to + disconnect on its own. `_terminate_and_wait` prefers this for a postgres handle so tearing + one down while its own dependents still hold connections does not have to eat the full + kill escalation every time. + """ + ... + + def wait(self, timeout: float) -> bool: + """Blocks up to `timeout` seconds for the process to exit. Returns whether it did.""" + ... + + def kill(self) -> None: ... + class ProcessRunner(Protocol): def __call__( @@ -756,6 +782,77 @@ def captured_output(self) -> str: def terminate(self) -> None: self.popen.terminate() + def interrupt(self) -> None: + self.popen.send_signal(signal.SIGINT) + + def wait(self, timeout: float) -> bool: + try: + self.popen.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + + def kill(self) -> None: + self.popen.kill() + + +_TERMINATE_WAIT_SECONDS = 5.0 +# Q12, p6-review-r2/-r3: rabbitmq's own broker shutdown routinely exceeds the 5s default — used +# only for the pre-`datadir_copy`-snapshot terminate, where a SIGKILL mid-mnesia-write would seal +# a corrupt baseline every world then restores from. R5, p6-review-r4: not a 30s ceiling — a +# wedged broker costs this value TWICE (`_terminate_and_wait` re-applies it to the post-kill() +# reap wait too), so the worst case at that call site is 60s. +_RABBITMQ_TERMINATE_WAIT_SECONDS = 30.0 + + +def _terminate_and_wait( + handle: SpawnedProcess, *, timeout: float = _TERMINATE_WAIT_SECONDS, + prefer_interrupt: bool = False, +) -> None: + """M7, p6-review-r1: `terminate()` alone only sends SIGTERM — postgres treats it as a SMART + shutdown that may not have even started, let alone finished, so the very next line at several + call sites (a `copytree` baseline snapshot, an `rmtree` before a port is reused) used to run + against a data directory a still-live server could still be writing to, or tried to rebind a + port the dying process had not yet released. Escalates to `kill()` only once `timeout` has + passed with no exit — a real SIGKILL, not a repeated SIGTERM, since a process that ignored the + first one is not going to notice a second. Never raises: a handle that is already gone, or + whose OS-level terminate/kill/wait itself errors, must not block whatever cleanup is calling + this (`close()`'s own idempotency, m10, is what actually made this matter). + + N12, p6-review-r2: `prefer_interrupt` swaps the FIRST signal from SIGTERM to SIGINT (postgres's + fast shutdown) — the caller decides this per-engine, since this function has no engine + knowledge of its own. + """ + try: + if prefer_interrupt: + handle.interrupt() + else: + handle.terminate() + except OSError: + pass + if handle.wait(timeout): + return + try: + handle.kill() + except OSError: + pass + # N13, p6-review-r2: the return here used to be discarded — a child still not reaped after + # SIGKILL (a wedged kernel wait, not something any further signal can fix) left no trace + # anywhere. Nothing past SIGKILL to escalate TO, but silently swallowing that fact is strictly + # worse than a log line a caller could act on (flag the sandbox instead of assuming cleanup + # succeeded). + if not handle.wait(timeout): + logger.warning("_terminate_and_wait: process did not exit even after kill()") + + +def _prefers_interrupt(manifest: EnvironmentBundleV2, process_name: str) -> bool: + """N12, p6-review-r2: postgres is the one engine whose fast-shutdown signal genuinely differs + from a plain SIGTERM (see `SpawnedProcess.interrupt`) — every other process (`source` or + redis/rabbitmq) just gets the ordinary `terminate()` path.""" + processes_by_name = {process.name: process for process in manifest.processes} + process = processes_by_name.get(process_name) + return isinstance(process, ManagedProcess) and process.engine is ManagedEngine.POSTGRES + def _default_log_chown(path: Path, uid: int, gid: int) -> None: os.chown(path, uid, gid) @@ -791,6 +888,11 @@ class SpawnedWorldProcess: handle: SpawnedProcess port: int world_index: int | None # None for a job-shared managed engine + # B2, p6-review-r1: the resolved spawn identity, carried on the handle so a caller that needs + # to run something ELSE (a seed file) as this same process's user does not have to re-resolve + # it a second time. `None` in the local-lane fallback, same as `spawn_uid`/`spawn_gid` above. + uid: int | None = None + gid: int | None = None # --- managed-engine launch commands --------------------------------------------------------- @@ -812,9 +914,10 @@ def postgres_bootstrap_argv( def postgres_daemon_argv(*, data_dir: Path, port: int) -> list[str]: - return [ - "postgres", "-D", str(data_dir), "-p", str(port), "-k", str(data_dir), "-h", "localhost", - ] + # n1, p6-review-r1: every connection this module makes is TCP `-h localhost` (seed, sentinel, + # canary, probes) — nothing ever dials the unix socket, so `-k ""` closes that listening + # surface entirely instead of leaving it open at `data_dir` unused. + return ["postgres", "-D", str(data_dir), "-p", str(port), "-k", "", "-h", "localhost"] def redis_daemon_argv(*, data_dir: Path, port: int) -> list[str]: @@ -828,11 +931,72 @@ def rabbitmq_daemon_argv() -> list[str]: return ["rabbitmq-server"] +def rabbitmq_enabled_plugins_text() -> str: + """M8, p6-review-r1: the Erlang term-list format `rabbitmq-server` reads at boot to decide + which plugins are on. `rabbitmq_daemon_env` sets node/auth/data-dir env vars only — nothing + turns the management app (the HTTP API `rabbitmqadmin` seeding and the sentinel/canary queue- + depth reads both depend on) on; without this file it is off and every one of those calls + connection-refuses.""" + return "[rabbitmq_management].\n" + + +def rabbitmq_conf_text(*, management_port: int, credentials: EngineCredentials) -> str: + """N5, p6-review-r2 (MAJOR): a BARE `rabbitmq-server` (§0: no Docker in the sandbox) reads + `default_user`/`default_pass` from THIS file — `RABBITMQ_DEFAULT_USER`/`RABBITMQ_DEFAULT_PASS` + (`rabbitmq_daemon_env`, below) are a convention the official Docker image's ENTRYPOINT + translates onto these same keys; nothing performs that translation for a directly-`exec`'d + `rabbitmq-server`, so without this the node initializes with the built-in `guest` account + instead of the catalog's declared `harness` role (§2b) — every rabbitmq call this module makes + (seed, sentinel, canary) authenticates as `harness` and would 401 against a node that never + heard of it. Q4, p6-review-r3: no `loopback_users` line here — `loopback_users = none` does + the OPPOSITE of what a prior version of this docstring claimed (it widens `guest`'s reach to + the whole network, RabbitMQ's own default `[guest]` restricts it to loopback already); with + `default_user = harness` a fresh node never creates a `guest` account at all, so the setting + buys nothing either way and is left out rather than left wrong. + + Also pins the management HTTP listener to this module's own `+10000`-offset port + (`_rabbitmq_management_port`) — the plugin's own built-in default is 15672, which sits inside + §2b's per-world process port band `[15000,15799]` and would alias an allocated process port; + binding `127.0.0.1` matches every other engine, which is localhost-only in V1. + """ + return ( + f"default_user = {credentials.username}\n" + f"default_pass = {credentials.password}\n" + f"management.tcp.port = {management_port}\n" + "management.tcp.ip = 127.0.0.1\n" + ) + + def rabbitmq_daemon_env( *, data_dir: Path, port: int, credentials: EngineCredentials ) -> dict[str, str]: + """`RABBITMQ_DEFAULT_USER`/`RABBITMQ_DEFAULT_PASS` are kept here as harmless redundancy (they + are exactly what the Docker image's entrypoint would consume, if the snapshot ever turned out + to wrap one after all — N5's own open question for the snapshot owner) — `rabbitmq_conf_text` + above is the credential source a BARE server actually reads. + + Q2, p6-review-r3 (MAJOR); R2, p6-review-r4: both `RABBITMQ_MNESIA_DIR` (a fixed, node-name-free + path) and `RABBITMQ_MNESIA_BASE` (`data_dir` itself) are set. `MNESIA_DIR` wins for the data + path, so a `datadir_copy` baseline snapshotted at world 0's port no longer sits under a path + only world 0's port-derived node name would look in. `MNESIA_BASE` still has to stay set: + rabbitmq derives OTHER paths from the base, not the dir — `RABBITMQ_PLUGINS_EXPAND_DIR` among + them — so dropping it moves those outside `data_dir`, to the installation default the spawn + user cannot write, and every rabbitmq spawn fails to boot (bootstrap included). + + KNOWN DEFECT (R1, p6-review-r4 — disclosed, not redesigned this phase): the mnesia DATA PATH + above is node-name-free, but the mnesia SCHEMA on disk is not — `schema.DAT` and rabbitmq's own + cluster-status file both record the node name (`RABBITMQ_NODENAME` below) the tables were + created under, and per-world node names must stay distinct (every world's broker registers with + the same epmd in the same sandbox). A `datadir_copy` baseline therefore boots cleanly in world 0 + only; every other world points its node at a directory holding world 0's node-bound schema, + which the broker will not adopt — expect a boot/readiness failure surfacing as + `depends_on_timeout` from `_wait_for_store_ready`, not a silent empty node. Recorded + follow-up: replace the datadir copy with a definitions-export/import restore through the + existing `default_rabbitmq_definitions_importer` seam, which is per-world safe by + construction.""" return { "RABBITMQ_NODE_PORT": str(port), + "RABBITMQ_MNESIA_DIR": str(data_dir / "mnesia"), "RABBITMQ_MNESIA_BASE": str(data_dir), "RABBITMQ_LOG_BASE": str(data_dir), "RABBITMQ_DEFAULT_USER": credentials.username, @@ -862,13 +1026,38 @@ def spawn_managed_process( runs in it (F1, p5-round1-review) — otherwise a daemon spawned under that user could not even write its own data directory, which the harness (running as `svc-control`) created. """ - data_dir.mkdir(parents=True, exist_ok=True) resolved_user = _resolve_process_user( process.user, resolver=user_resolver, require=require_declared_user, process_name=process.name, stage="spawn", ) - if resolved_user is not None: - chown(data_dir, resolved_user.pw_uid, resolved_user.pw_gid) + try: + data_dir.mkdir(parents=True, exist_ok=True) + # postgres demands mode 0700 on the directory itself regardless of engine; harmless for + # redis/rabbitmq, so applied uniformly rather than special-cased per engine. N14, + # p6-review-r2: BEFORE the chown below, not after — `svc-control` still owns `data_dir` at + # this point and the chmod is free; once ownership has moved to the engine's own uid, the + # SAME call would raise `PermissionError` for a `svc-control` that can chown but is not + # itself root. + data_dir.chmod(0o700) + if resolved_user is not None: + # B6, p6-review-r1: a single top-level `chown` was fine for a FRESH, empty data_dir + # (the daemon's own first-boot `initdb` then creates everything else as `spawn_uid` + # itself) but wrong the moment this same call runs against a data_dir a COPY just + # populated (`freeze_baseline`'s snapshot restore, `_seal_world_store`'s `datadir_copy` + # reseal) — the copy runs as the provisioner, so every file underneath stayed + # provisioner-owned and postgres refuses to start against a data directory it does not + # fully own. + _chown_tree(data_dir, uid=resolved_user.pw_uid, gid=resolved_user.pw_gid, chown=chown) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2: §4.6 — "engine/process/filesystem failures during provisioning -> + # infrastructure." mkdir/chmod/chown on this engine's own data directory used to raise + # bare, giving `provision()`'s caller nothing to map; every failure this module's own + # data-dir setup can hit is the same infrastructure class B5 already typed for the store + # seams themselves. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", + process=process.name, + ) from exc spawn_uid = resolved_user.pw_uid if resolved_user is not None else None spawn_gid = resolved_user.pw_gid if resolved_user is not None else None @@ -920,7 +1109,43 @@ def spawn_managed_process( "spawn", "spawn_failed", "rabbitmq requires generated credentials", process=process.name, ) + # M8, p6-review-r1: written fresh on every spawn (job bootstrap AND every world's own + # instance) — cheap, and means a `datadir_copy` restore can never carry a stale plugin/ + # port config forward from whatever was on disk when the baseline was snapshotted. + plugins_path = data_dir / "enabled_plugins" + conf_path = data_dir / "rabbitmq.conf" + try: + plugins_path.write_text(rabbitmq_enabled_plugins_text(), encoding="utf-8") + # N5, p6-review-r2: 0600 on CREATE — this file carries the generated password in + # cleartext (`rabbitmq_conf_text`). Plain `O_TRUNC`, not `O_EXCL`: unlike the postgres + # pwfile, `data_dir` legitimately already HAS this file on a `datadir_copy` restore + # (it was copied in from the baseline snapshot along with everything else), and the + # mode set at CREATE time already carried through that copy (`shutil.copytree`'s + # default `copy2` preserves permission bits) — `O_EXCL` would just raise + # `FileExistsError` on every world after the first. + fd = os.open(conf_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(rabbitmq_conf_text( + management_port=_rabbitmq_management_port(port), credentials=credentials, + )) + if resolved_user is not None: + chown(plugins_path, resolved_user.pw_uid, resolved_user.pw_gid) + chown(conf_path, resolved_user.pw_uid, resolved_user.pw_gid) + except OSError as exc: + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: writing rabbitmq config: {exc}", + process=process.name, + ) from exc env.update(rabbitmq_daemon_env(data_dir=data_dir, port=port, credentials=credentials)) + env["RABBITMQ_ENABLED_PLUGINS_FILE"] = str(plugins_path) + # N6, p6-review-r2: the FULL path, extension included. Modern RabbitMQ (the catalog's + # 3.13) documents `RABBITMQ_CONFIG_FILE` carrying `.conf` itself and unambiguously accepts + # it that way — the pre-3.7 classic-config "the server appends .conf for you" behavior + # this used to depend on is not how the catalog's version works, and the failure was + # silent: the file on disk is `rabbitmq.conf`, the env var said `rabbitmq`, and if the + # append never happens the server falls back to defaults — including the bare management + # port 15672, which sits inside §2b's own per-world port band. + env["RABBITMQ_CONFIG_FILE"] = str(conf_path) argv = rabbitmq_daemon_argv() else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. raise ProcessRuntimeError( @@ -934,7 +1159,8 @@ def spawn_managed_process( except FileNotFoundError as exc: raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc return SpawnedWorldProcess( - process_name=process.name, handle=handle, port=port, world_index=None + process_name=process.name, handle=handle, port=port, world_index=None, + uid=spawn_uid, gid=spawn_gid, ) @@ -960,13 +1186,21 @@ def spawn_source_process( process running as anyone but the harness's own uid could not write into its own `{{WORLD_ DIR}}` at all, since the harness (as `svc-control`) is the one that just created it. """ - world_dir.mkdir(parents=True, exist_ok=True) resolved_user = _resolve_process_user( process.user, resolver=user_resolver, require=require_declared_user, process_name=process.name, stage="spawn", ) - if resolved_user is not None: - chown(world_dir, resolved_user.pw_uid, resolved_user.pw_gid) + try: + world_dir.mkdir(parents=True, exist_ok=True) + if resolved_user is not None: + chown(world_dir, resolved_user.pw_uid, resolved_user.pw_gid) + except OSError as exc: + # N9, p6-review-r2: same class as `spawn_managed_process`'s own data-dir setup boundary — + # a permission error creating/chowning this process's `{{WORLD_DIR}}` used to raise bare. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: preparing {world_dir}: {exc}", + process=process.name, + ) from exc rendered = render_environment( process, world_index=world_index, @@ -990,7 +1224,9 @@ def spawn_source_process( raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc port = port_plan.port_for(process.name, world_index) return SpawnedWorldProcess( - process_name=process.name, handle=handle, port=port, world_index=world_index + process_name=process.name, handle=handle, port=port, world_index=world_index, + uid=resolved_user.pw_uid if resolved_user is not None else None, + gid=resolved_user.pw_gid if resolved_user is not None else None, ) @@ -1056,14 +1292,29 @@ def _tcp_probe(host: str, port: int, *, timeout: float = 0.75) -> bool: return False -def _probe_http(host: str, port: int, path: str | None, *, timeout: float = 1.0) -> bool: +def _probe_http( + host: str, port: int, path: str | None, *, user: str | None = None, + password: str | None = None, timeout: float = 1.0, +) -> bool: + """N7, p6-review-r2: `user`/`password`, when both given, ride along as a Basic-auth header — + added for the rabbitmq management listener probe (`_wait_for_store_ready`), which otherwise + 401s on every attempt regardless of whether the listener is actually up.""" url = f"http://{host}:{port}/{(path or '').lstrip('/')}" try: opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - with opener.open(url, timeout=timeout) as response: + request = urllib.request.Request(url) + if user is not None and password is not None: + credential_bytes = f"{user}:{password}".encode("utf-8") + request.add_header( + "Authorization", "Basic " + base64.b64encode(credential_bytes).decode() + ) + with opener.open(request, timeout=timeout) as response: response.read(256) return 200 <= response.status < 400 - except (OSError, urllib.error.URLError, urllib.error.HTTPError): + except Exception: + # Q6, p6-review-r3: `http.client.HTTPException` (a malformed status line, a truncated + # body) is not an `OSError` — a probe's contract is bool-not-raise, same broad form + # `_probe_postgres` already uses below. return False @@ -1109,7 +1360,7 @@ def default_capability_prober( if protocol is CapabilityProtocol.POSTGRES: return _probe_postgres(host, port, user=user, password=password, dbname=dbname) if protocol is CapabilityProtocol.HTTP: - return _probe_http(host, port, path) + return _probe_http(host, port, path, user=user, password=password) return _tcp_probe(host, port) @@ -1260,6 +1511,499 @@ def visit(name: str) -> None: return order +# --- §2c/§5 store command seams ----------------------------------------------------------------- +# +# One injectable seam per engine, mirroring `ProcessRunner`/`CapabilityProber`: real production +# code talks to the actual store, a test hands in a spy. `SqlRunner` alone carries the "SQL spy" +# tests assert TEMPLATE/DROP/CREATE/sentinel statements against — postgres is the only engine +# `no_sql_store` (§2e item 6) guarantees, so it is the only one whose statement-level shape this +# module commits to; redis/rabbitmq get the same command-injection treatment but a plainer one +# (`argv`, not a statement AST), matching how much less this contract pins about them (§2c: seeded +# "only if the repo ships seed state for them"). + + +class SqlRunner(Protocol): + def __call__( + self, *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, + ) -> list[tuple[Any, ...]]: ... + + +def default_sql_runner( + *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, +) -> list[tuple[Any, ...]]: + """psycopg-backed, import-guarded like `_probe_postgres` — never exercised in this module's + own test lane (no real postgres here), always injected by a test's SQL spy instead. + + N8, p6-review-r2 (MAJOR): `read_only`, when set, puts the session into a read-only default + transaction mode before running `statement` — the ONLY thing that made `sentinel.query` (and + the baseline row-count reads it shares this code path with) different from an arbitrary write + was convention: it runs as `harness`, the role `initdb -U harness` makes the postgres + SUPERUSER, over a plain autocommit session, executing customer-authored content from an + untrusted repo verbatim. `world-handle-interface.md` v3.4's own `query()` runs on a `SET + TRANSACTION READ ONLY` connection and calls that "the guard" — this is the provisioner's side + of the same guard, for every call site that is supposed to be a read (`_call_sql`'s own + `read_only=True` callers). + """ + import psycopg # type: ignore[import-not-found] + + with psycopg.connect( + host=host, port=port, user=user, password=password, dbname=dbname, autocommit=True, + ) as connection: + if read_only: + connection.execute("SET default_transaction_read_only = on") + cursor = connection.execute(statement) + try: + return cursor.fetchall() + except psycopg.ProgrammingError: + return [] # a DDL/utility statement (CREATE DATABASE, ALTER DATABASE, ...) has no rows. + + +class RedisCommandRunner(Protocol): + def __call__(self, *, host: str, port: int, command: Sequence[str]) -> Any: ... + + +def default_redis_command_runner(*, host: str, port: int, command: Sequence[str]) -> Any: + import redis # type: ignore[import-not-found] + + client = redis.Redis(host=host, port=port) + try: + return client.execute_command(*command) + finally: + client.close() + + +class RabbitmqQueueInspector(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str + ) -> int: ... + + +class RabbitmqQueueDeclarer(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str, message: str, + ) -> None: ... + + +class RabbitmqQueueDeleter(Protocol): + """Q10, p6-review-r3: no remaining caller — N15 (p6-review-r2) moved canary cleanup to the + reset-from-baseline path this seam predates. Retained deliberately (constructor param, + `SpawnContext` field) as the explicit-delete seam a future canary strategy may still want, + rather than ripped out and re-added from scratch.""" + + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str, + ) -> None: ... + + +# --- B5, p6-review-r1: typed failures at the store-command seams ----------------------------- +# +# Every call below runs AFTER the store's own readiness has already been established (B4's wait, +# or an already-running job-shared engine) — a driver exception here is the store rejecting or +# erroring on the provisioner's OWN statement, never a connection race, so it is §2f's +# `store_statement_failed` (infrastructure, retryable — v1.10), never `depends_on_timeout` (that +# code belongs to the readiness WAIT itself, which `default_capability_prober` already degrades +# to a bool instead of raising) and never `seed_failed` (reserved for the customer's own +# migration/seed CONTENT, applied through `sync_run`'s exit code, not through these seams at +# all). One wrapper per seam, matching the existing one-Protocol-per-engine-concern split. + + +def _call_sql( + sql_runner: SqlRunner, *, stage: str, process_name: str, host: str, port: int, user: str, + password: str, dbname: str, statement: str, read_only: bool = False, +) -> list[tuple[Any, ...]]: + try: + return sql_runner( + host=host, port=port, user=user, password=password, dbname=dbname, statement=statement, + read_only=read_only, + ) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued statement: {exc}", + process=process_name, + ) from exc + + +def _call_redis( + redis_runner: RedisCommandRunner, *, stage: str, process_name: str, host: str, port: int, + command: Sequence[str], +) -> Any: + try: + return redis_runner(host=host, port=port, command=command) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued command: {exc}", + process=process_name, + ) from exc + + +def _call_rabbitmq( + rabbitmq_inspector: RabbitmqQueueInspector, *, stage: str, process_name: str, host: str, + port: int, credentials: EngineCredentials, queue: str, +) -> int: + try: + return rabbitmq_inspector(host=host, port=port, credentials=credentials, queue=queue) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued queue inspection: {exc}", + process=process_name, + ) from exc + + +_RABBITMQ_DEPTH_READ_ATTEMPTS = 3 +_RABBITMQ_DEPTH_READ_INTERVAL_SECONDS = 0.05 + + +def _call_rabbitmq_with_retry( + rabbitmq_inspector: RabbitmqQueueInspector, *, stage: str, process_name: str, host: str, + port: int, credentials: EngineCredentials, queue: str, accept: Callable[[int], bool], + sleep: Callable[[float], None] = time.sleep, +) -> int: + """N15, p6-review-r2 (MINOR): the management API's `messages` field is fed by the node's own + stats collector (`collect_statistics_interval`, default 5000ms) — a depth read immediately + after a publish/reset can report a stale value in EITHER direction: 0 when a message really + is there (the direction that falsely PASSES an isolation check), or a leftover count when it + really is gone (the direction that falsely FAILS an `expected_depth` sentinel). `accept` is + the caller's own success predicate; retried a bounded few times before settling for whatever + the last read reported, so a genuinely wrong result is never masked by retrying forever — only + the collector's known lag window is absorbed. + """ + depth = 0 + for attempt in range(_RABBITMQ_DEPTH_READ_ATTEMPTS): + depth = _call_rabbitmq( + rabbitmq_inspector, stage=stage, process_name=process_name, host=host, port=port, + credentials=credentials, queue=queue, + ) + if accept(depth): + return depth + if attempt < _RABBITMQ_DEPTH_READ_ATTEMPTS - 1: + sleep(_RABBITMQ_DEPTH_READ_INTERVAL_SECONDS) + return depth + + +def _call_rabbitmq_action( + fn: Callable[..., None], *, stage: str, process_name: str, action: str, **kwargs: Any, +) -> None: + """Same B5 typing as `_call_rabbitmq`, for the write-side canary declare/publish call + `_run_canary_probe` makes through `context.rabbitmq_declare` — the m8 canary's OWN statement + against a store that has already passed readiness, exactly the class of call B5 covers. (Q10, + p6-review-r3: `context.rabbitmq_delete` is no longer one of these — N15 moved canary cleanup + to the reset-from-baseline path; the seam itself is kept, see `RabbitmqQueueDeleter`.)""" + try: + fn(**kwargs) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected the provisioner's canary {action}: {exc}", + process=process_name, + ) from exc + + +def _rabbitmq_management_port(amqp_port: int) -> int: + """§2b's catalog/port formula fixes only the AMQP listener port; the management HTTP API + (needed to seed via `rabbitmqadmin` and to read queue depth for a sentinel) has no formula of + its own. A fixed `+10000` offset is a defensible, deterministic V1 default — `14000..14099` / + `15000..15799` (§2b) shifted by it lands at `24000..24099` / `25000..25799`, outside every + band this contract reserves, so it can never alias a different process's allocated port. Same + provisioner-internal status as `postgres_daemon_argv` et al.: swappable, never fixed by the + contract, never exercised against a real broker in this module's own test lane. + """ + return amqp_port + 10000 + + +def _rabbitmq_auth_header(credentials: EngineCredentials) -> str: + credential_bytes = f"{credentials.username}:{credentials.password}".encode("utf-8") + return "Basic " + base64.b64encode(credential_bytes).decode() + + +def default_rabbitmq_queue_inspector( + *, host: str, port: int, credentials: EngineCredentials, queue: str +) -> int: + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + request = urllib.request.Request(url) + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + try: + with urllib.request.urlopen(request, timeout=5.0) as response: + payload = json.loads(response.read()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + # m8, p6-review-r1: a nonexistent queue is what "empty"/"absent" LOOKS like at this + # endpoint — the management API's own way of saying so is a 404, not an error the + # conformance gate's "never raises" promise should have to survive by accident. + return 0 + raise + return int(payload.get("messages", 0)) + + +def default_rabbitmq_queue_declare_and_publish( + *, host: str, port: int, credentials: EngineCredentials, queue: str, message: str, +) -> None: + """m8, p6-review-r1: the conformance canary previously only ever INSPECTED a rabbitmq queue — + since nothing ever created one, world 1's "is it visible" check compared against a queue that + never existed anywhere, a vacuous pass. Declares the reserved canary queue in world 0 for + real via the management API (never AMQP directly — same HTTP-only seam as every other + rabbitmq call this module makes) and publishes one message into it, so world 1's read is + checking something that could actually leak. + """ + management_port = _rabbitmq_management_port(port) + auth = _rabbitmq_auth_header(credentials) + + declare_url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + declare_request = urllib.request.Request( + declare_url, + data=json.dumps({"durable": False, "auto_delete": True}).encode("utf-8"), + method="PUT", + ) + declare_request.add_header("Content-Type", "application/json") + declare_request.add_header("Authorization", auth) + with urllib.request.urlopen(declare_request, timeout=5.0): + pass + + # The default exchange's routing key IS the queue name — the standard way the management + # API's own `publish` endpoint targets a specific queue without declaring a binding first. + publish_url = f"http://{host}:{management_port}/api/exchanges/%2F/amq.default/publish" + publish_request = urllib.request.Request( + publish_url, + data=json.dumps({ + "properties": {}, "routing_key": queue, "payload": message, + "payload_encoding": "string", + }).encode("utf-8"), + method="POST", + ) + publish_request.add_header("Content-Type", "application/json") + publish_request.add_header("Authorization", auth) + with urllib.request.urlopen(publish_request, timeout=5.0): + pass + + +def default_rabbitmq_queue_delete( + *, host: str, port: int, credentials: EngineCredentials, queue: str, +) -> None: + """Cleanup half of the canary (m8) — best-effort: a 404 means it is already gone (e.g. the + world-0 reset that runs right after already wiped it, since `datadir_copy` is rabbitmq's only + legal strategy), which is the desired end state, not a failure.""" + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + request = urllib.request.Request(url, method="DELETE") + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + try: + with urllib.request.urlopen(request, timeout=5.0): + pass + except urllib.error.HTTPError as exc: + if exc.code != 404: + raise + + +class RabbitmqDefinitionsImporter(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, file: Path, + ) -> None: ... + + +def default_rabbitmq_definitions_importer( + *, host: str, port: int, credentials: EngineCredentials, file: Path, +) -> None: + """N17, p6-review-r2 (MINOR): `rabbitmqadmin import ` used to require the + `rabbitmqadmin` binary specifically — served BY the management plugin at runtime (fetched + from the running node's own web UI), never installed by the `rabbitmq-server` package itself, + and not in §0's guaranteed-binary list (`python`, `node`, `git`, `ffmpeg`, plus the §2b engine + catalog). `rabbitmqadmin import` is documented as a thin wrapper over exactly this endpoint + (`POST /api/definitions`) — the same management HTTP API this module already talks to for + every other rabbitmq call (`default_rabbitmq_queue_inspector` et al.), so seeding no longer + depends on a binary the snapshot might not ship at all. + """ + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/definitions" + request = urllib.request.Request(url, data=file.read_bytes(), method="POST") + request.add_header("Content-Type", "application/json") + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + with urllib.request.urlopen(request, timeout=10.0): + pass + + +# --- §2c seed application ------------------------------------------------------------------- + + +def postgres_seed_argv(*, port: int, dbname: str, user: str, file: Path) -> list[str]: + return [ + "psql", "-h", "localhost", "-p", str(port), "-U", user, "-d", dbname, + "-v", "ON_ERROR_STOP=1", "-f", str(file), + ] + + +def postgres_seed_env(credentials: EngineCredentials) -> dict[str, str]: + return {"PGPASSWORD": credentials.password} + + +def redis_seed_argv(*, port: int) -> list[str]: + """No `-f`/script flag exists on `redis-cli`; a seed file is plain commands, one per line, + fed over stdin — the documented way `redis-cli` accepts a batch of ordinary commands (as + opposed to `--pipe`, which expects pre-encoded RESP, not this module's business to generate). + """ + return ["redis-cli", "-h", "localhost", "-p", str(port)] + + +def apply_seed_file( + engine: ManagedEngine, + file: Path, + *, + port: int, + dbname: str, + credentials: EngineCredentials | None, + process_name: str, + sync_run: Callable[..., subprocess.CompletedProcess], + user: int | None = None, + group: int | None = None, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, +) -> None: + """Applies one migration/seed file, per §2c: "applied in listed order." `postgres` shells out + to a psql-style command (`-f`, so a large schema file streams rather than loading into this + process); `redis`/`rabbitmq` use their own bulk-load mechanisms — three small per-engine + branches, mirroring `postgres_daemon_argv` et al.'s split, so a fourth catalog engine needs + only one new branch here, never a rewrite of the caller. `sync_run` is `SpawnContext.sync_run` + (or a test's fake) — the same "run one synchronous step, check its exit code" seam + `build_process_tree`'s build steps and `spawn_managed_process`'s postgres bootstrap already + use. Raises on the first failing file (`ProcessRuntimeError`, stage="seed"), never partially + proceeds silently past one — same rule `build_process_tree` holds for `build_commands`. + + B2, p6-review-r1: `user`/`group` are the store's OWN declared spawn identity (`svc-data`), + never left to default to the provisioner's own uid — `psql -f` honors backslash meta-commands + (`\\!`, `\\copy ... program`), so a migration/seed file is a customer-authored-content + execution path exactly like `build_commands`, which already drops privilege the same way. + """ + if engine is ManagedEngine.POSTGRES: + if credentials is None: + raise ProcessRuntimeError( + "seed", "internal_missing_credentials", + "postgres requires generated credentials to seed", process=process_name, + ) + argv = postgres_seed_argv(port=port, dbname=dbname, user=credentials.username, file=file) + # F12's own rule (§2b's closed env list), reapplied here: `env=` fully REPLACES a child's + # environment rather than extending the caller's — passing just `{"PGPASSWORD": ...}` + # would drop `PATH` entirely, and `psql` living anywhere outside `subprocess`'s POSIX + # fallback path (`/bin:/usr/bin`) — a homebrew or venv install, commonly — would silently + # fail to exec. + env = {**_allowlisted_ambient_env(os.environ), **postgres_seed_env(credentials)} + result = sync_run(argv, capture_output=True, text=True, env=env, user=user, group=group) + elif engine is ManagedEngine.REDIS: + argv = redis_seed_argv(port=port) + result = sync_run( + argv, capture_output=True, text=True, input=file.read_text(encoding="utf-8"), + user=user, group=group, + ) + elif engine is ManagedEngine.RABBITMQ: + if credentials is None: + raise ProcessRuntimeError( + "seed", "internal_missing_credentials", + "rabbitmq requires generated credentials to seed", process=process_name, + ) + # N17, p6-review-r2: seeded over the management HTTP API directly, never through a + # subprocess/`sync_run` — no `rabbitmqadmin` binary dependency (`user`/`group` are moot + # here for the same reason: an HTTP call has no OS-level identity to drop). + try: + rabbitmq_import(host="localhost", port=port, credentials=credentials, file=file) + except Exception as exc: + raise ProcessRuntimeError( + "seed", "seed_failed", f"{file}: {exc}", process=process_name, + ) from exc + return + else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + raise ProcessRuntimeError( + "seed", "seed_failed", f"unknown engine {engine!r}", process=process_name + ) + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "seed", "seed_failed", + f"{file}: exited {result.returncode}" + (f": {stderr}" if stderr else ""), + process=process_name, + ) + + +def apply_store_seed( + store: StoreEntry, + *, + engine: ManagedEngine, + bundle_dir: Path, + port: int, + dbname: str, + credentials: EngineCredentials | None, + process_name: str, + sync_run: Callable[..., subprocess.CompletedProcess], + user: int | None = None, + group: int | None = None, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, +) -> None: + """§2c: "migrations then seed_files... applied in listed order" — migrations always precede + seed_files, regardless of how many files either list holds, and each list keeps its own + authored order (never sorted).""" + for relative_path in (*store.migrations, *store.seed_files): + apply_seed_file( + engine, bundle_dir / relative_path, port=port, dbname=dbname, credentials=credentials, + process_name=process_name, sync_run=sync_run, user=user, group=group, + rabbitmq_import=rabbitmq_import, + ) + + +# --- §2c sentinel checking --------------------------------------------------------------------- + + +def check_sentinel( + store: StoreEntry, + *, + engine: ManagedEngine, + host: str, + port: int, + dbname: str | None, + credentials: EngineCredentials | None, + sql_runner: SqlRunner, + redis_runner: RedisCommandRunner, + rabbitmq_inspector: RabbitmqQueueInspector, + process_name: str = "", + stage: str = "sentinel", +) -> bool: + """§2c/§4.2: "a read-only check plus its exact expected value (string compare)." Dispatches on + the sentinel's own implied shape (`Sentinel.implied_engine`), which the model layer already + guarantees matches the store's engine (`sentinel_shape_mismatch`) — this only has to act on + it, not re-verify it. `process_name`/`stage` are for B5's typed-failure wrapper only (default + to the empty string / "sentinel" so every pre-existing direct caller keeps working unchanged). + """ + sentinel = store.sentinel + if engine is ManagedEngine.POSTGRES: + if credentials is None or dbname is None: + return False + rows = _call_sql( + sql_runner, stage=stage, process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement=sentinel.query, # type: ignore[arg-type] + read_only=True, # N8, p6-review-r2: customer-authored content, §2c calls this a read. + ) + actual = str(rows[0][0]) if rows and rows[0] else None + return actual == sentinel.expected + if engine is ManagedEngine.REDIS: + value = _call_redis( + redis_runner, stage=stage, process_name=process_name, + host=host, port=port, command=["GET", sentinel.key], # type: ignore[list-item] + ) + actual = value.decode() if isinstance(value, bytes) else (None if value is None else str(value)) + return actual == sentinel.expected + if engine is ManagedEngine.RABBITMQ: + if credentials is None: + return False + depth = _call_rabbitmq_with_retry( + rabbitmq_inspector, stage=stage, process_name=process_name, + host=host, port=port, credentials=credentials, queue=sentinel.queue, # type: ignore[arg-type] + accept=lambda value: value == sentinel.expected_depth, # N15, p6-review-r2. + ) + return depth == sentinel.expected_depth + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + # --- per-world spawn orchestration -------------------------------------------------------------- @@ -1291,6 +2035,19 @@ class SpawnContext: require_declared_user: bool = False chown: Callable[[Path, int, int], None] = _default_chown build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS + # §2c/§5 store command seams (below) — defaulted so every existing `SpawnContext(...)` call + # in P5's own tests keeps working unchanged; Phase 6 callers override them with spies. + sql_runner: SqlRunner = default_sql_runner + redis_runner: RedisCommandRunner = default_redis_command_runner + rabbitmq_inspector: RabbitmqQueueInspector = default_rabbitmq_queue_inspector + # m8, p6-review-r1: the conformance canary's rabbitmq branch (below `_run_canary_probe`) + # needs to actually create the reserved queue, not just read one that was never declared. + rabbitmq_declare: RabbitmqQueueDeclarer = default_rabbitmq_queue_declare_and_publish + rabbitmq_delete: RabbitmqQueueDeleter = default_rabbitmq_queue_delete + # N17, p6-review-r2: seeding no longer shells out to `rabbitmqadmin` — the same management + # HTTP API seam every other rabbitmq call in this context already uses. + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer + bundle_dir: Path | None = None @dataclass @@ -1324,53 +2081,63 @@ def spawn_world( processes_by_name = {process.name: process for process in manifest.processes} handles: dict[str, SpawnedWorldProcess] = dict(shared_handles or {}) - for name in _topological_order(manifest): - if name in handles: - continue # a job-shared managed engine already running from an earlier world. - process = processes_by_name[name] - for dependency_name in process.depends_on: - wait_for_dependency( - manifest, - dependency_name, - world_index=world_index, - port_plan=context.port_plan, - spawned=handles[dependency_name], - credentials=context.credentials, - prober=context.prober, - ) - if isinstance(process, ManagedProcess): - data_dir = managed_engine_data_dir( - context.work_directory, - name, - world_index=None if context.port_plan.is_job_shared(name) else world_index, - ) - handle = spawn_managed_process( - process, - port=context.port_plan.port_for(name, world_index), - data_dir=data_dir, - credentials=context.credentials.get(name), - runner=context.runner, - sync_run=context.sync_run, - user_resolver=context.user_resolver, - require_declared_user=context.require_declared_user, - chown=context.chown, - ) - else: - handle = spawn_source_process( - process, - build_dir=build_tree_dir(context.work_directory, name), - world_dir=world_scratch_dir(context.work_directory, world_index, name), - world_index=world_index, - port_plan=context.port_plan, - configuration_addresses=configuration_addresses, - secret_values=context.secret_values, - secret_purposes=context.secret_purposes, - runner=context.runner, - user_resolver=context.user_resolver, - require_declared_user=context.require_declared_user, - chown=context.chown, - ) - handles[name] = handle + try: + for name in _topological_order(manifest): + if name in handles: + continue # a job-shared managed engine already running from an earlier world. + process = processes_by_name[name] + for dependency_name in process.depends_on: + wait_for_dependency( + manifest, + dependency_name, + world_index=world_index, + port_plan=context.port_plan, + spawned=handles[dependency_name], + credentials=context.credentials, + prober=context.prober, + ) + if isinstance(process, ManagedProcess): + data_dir = managed_engine_data_dir( + context.work_directory, + name, + world_index=None if context.port_plan.is_job_shared(name) else world_index, + ) + handle = spawn_managed_process( + process, + port=context.port_plan.port_for(name, world_index), + data_dir=data_dir, + credentials=context.credentials.get(name), + runner=context.runner, + sync_run=context.sync_run, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + else: + handle = spawn_source_process( + process, + build_dir=build_tree_dir(context.work_directory, name), + world_dir=world_scratch_dir(context.work_directory, world_index, name), + world_index=world_index, + port_plan=context.port_plan, + configuration_addresses=configuration_addresses, + secret_values=context.secret_values, + secret_purposes=context.secret_purposes, + runner=context.runner, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + handles[name] = handle + except BaseException as exc: + # N4, p6-review-r2 (MAJOR): a `depends_on_timeout`/`spawn_failed` partway through this + # world's own process list used to drop every ALREADY-spawned process of the SAME world on + # the floor — nothing held it, so a caller's later `close()`/retry could `rmtree` a data + # directory, or attempt a fresh spawn, out from under a still-live sibling process. + # `handles` (this world's own accumulated dict, seeded from `shared_handles`) is attached + # to the exception so the caller can publish/terminate it instead. + exc.partial_handles = handles # type: ignore[attr-defined] + raise return WorldSpawnResult(handles=handles, endpoints=endpoints) @@ -1468,46 +2235,1846 @@ async def healthy( return is_healthy -__all__ = [ - "CapabilityProber", - "EngineCredentials", - "EnvironmentRuntime", - "PopenProcess", - "PortPlan", - "ProcessRunner", - "ProcessRuntimeError", - "RuntimeEndpoint", - "RuntimeState", - "SpawnContext", - "SpawnedProcess", - "SpawnedWorldProcess", - "WorldSpawnResult", - "build_endpoints", - "build_process_tree", - "build_process_trees", - "build_tree_dir", - "configuration_addresses_from_endpoints", - "default_capability_prober", - "default_process_runner", - "default_user_resolver", - "generate_engine_credentials", - "healthy", - "managed_engine_data_dir", - "new_runtime_id", - "plan_ports", - "probe_runtime_health", - "rabbitmq_daemon_argv", - "rabbitmq_daemon_env", - "redis_daemon_argv", - "render_capability_address", - "render_environment", - "render_template", - "postgres_bootstrap_argv", - "postgres_daemon_argv", +# --- internal invariant codes ------------------------------------------------------------------- +# +# Neither is in §2f's closed table — both mark a precondition upstream layers already guarantee +# (a postgres/rabbitmq `ManagedProcess` always gets generated credentials; a store's backing +# service is always a `ManagedProcess`, per `bundle_v2`'s own `store_service_not_managed`), same +# status as `render_capability_address`'s `internal_missing_credentials` above — a bug to fix +# here, never a bundle defect the outbound seam needs a name for. +_INTERNAL_MISSING_CREDENTIALS = "internal_missing_credentials" +_INTERNAL_INVARIANT_VIOLATED = "internal_invariant_violated" + + +def _require_credentials( + credentials: EngineCredentials | None, *, stage: str, process_name: str +) -> EngineCredentials: + if credentials is None: + raise ProcessRuntimeError( + stage, _INTERNAL_MISSING_CREDENTIALS, + f"{process_name}: postgres/rabbitmq requires generated credentials but none were " + "supplied", process=process_name, + ) + return credentials + + +# --- §2c baseline: store lookup + naming ----------------------------------------------------- + + +def _store_by_process_name(manifest: EnvironmentBundleV2) -> dict[str, StoreEntry]: + """Every declared store, keyed by the `ManagedProcess.name` backing its capability — the + shape every baseline/clone/reset function below actually wants, one hop past what §2c's own + `seed.stores[].capability` names.""" + result: dict[str, StoreEntry] = {} + if manifest.seed is not None: + for store in manifest.seed.stores: + capability = manifest.capabilities[store.capability] + result[capability.service] = store + return result + + +def _template_database_name(process_name: str) -> str: + """§2c's own baseline databases are never `w` (§2b reserves that shape for per-world logical + DBs) — `-` is replaced since a process `name` may carry it (`^[a-z0-9][a-z0-9_-]*$`, §2b) but a + bare postgres identifier cannot without quoting.""" + return f"alk_baseline_{process_name.replace('-', '_')}" + + +def _datadir_copy_baseline_path(work_directory: Path, process_name: str) -> Path: + """A sibling of the live `/work/managed//` dir (§0), never nested inside it — `freeze_ + baseline` copies FROM the live dir INTO this one; nesting would make that copy recurse into + its own destination.""" + return _ensure_within( + work_directory / "managed" / f"{process_name}.baseline", work_directory, + process_name=process_name, stage="baseline", + ) + + +def _measure_postgres_row_counts( + *, host: str, port: int, credentials: EngineCredentials, dbname: str, sql_runner: SqlRunner, + process_name: str, +) -> dict[str, int]: + """Per-table row counts as the baseline stands right after seeding — `HostedWorld`'s own + `baseline_row_counts` input (`world/handle.py`'s `STATE_ROW_CAP` gate), measured once here + rather than re-measured at scenario time, so the cap it enforces cannot depend on what a + scenario already wrote to the table.""" + tables = _call_sql( + sql_runner, stage="baseline", process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement="SELECT tablename FROM pg_tables WHERE schemaname = 'public'", + read_only=True, # N8, p6-review-r2. + ) + counts: dict[str, int] = {} + for row in tables: + table = row[0] + # M1, p6-review-r1: `table` is the raw `pg_tables.tablename` — from the untrusted repo's + # own migrations. An unescaped double quote inside it used to close the identifier early + # and let anything after it execute as a second statement (psycopg3's simple-query + # protocol runs every statement in an unparameterized `execute()` call). + safe_table = table.replace('"', '""') + result = _call_sql( + sql_runner, stage="baseline", process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement=f'SELECT COUNT(*) FROM "{safe_table}"', + read_only=True, # N8, p6-review-r2. + ) + counts[table] = int(result[0][0]) if result and result[0] else 0 + return counts + + +# --- B4, p6-review-r1: readiness wait between a managed-engine spawn and the first statement -- + +_DEFAULT_STORE_READY_TIMEOUT_SECONDS = 30.0 +_DEFAULT_STORE_READY_INTERVAL_SECONDS = 0.25 + +_PROTOCOL_BY_ENGINE = { + ManagedEngine.POSTGRES: CapabilityProtocol.POSTGRES, + ManagedEngine.REDIS: CapabilityProtocol.REDIS, + ManagedEngine.RABBITMQ: CapabilityProtocol.AMQP, +} + + +def _wait_for_store_ready( + manifest: EnvironmentBundleV2, + process: ManagedProcess, + *, + port: int, + context: SpawnContext, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """B4, p6-review-r1: `wait_for_dependency` already makes every `depends_on` EDGE wait for its + dependency's readiness before touching it; a freeze/seal call site is not itself anyone's + declared dependent, so the very same class of race (the engine has just been `exec`'d and has + not finished booting) went unguarded at every place THIS module spawns a store immediately + before issuing its own first statement against it. Reuses the manifest's declared `readiness` + timeout for this process's capability when the bundle names one, else a bounded default — + `default_capability_prober` already degrades to a bare TCP probe without credentials, so this + never requires a real login to work. Raises `depends_on_timeout` (§2f) on exhaustion, the same + code `wait_for_dependency` uses for the identical class of failure. + """ + probes = _readiness_probes_for_process(manifest, process.name) + timeout = max( + (probe.timeout_seconds for probe in probes), default=_DEFAULT_STORE_READY_TIMEOUT_SECONDS + ) + interval = min( + (probe.interval_seconds for probe in probes), default=_DEFAULT_STORE_READY_INTERVAL_SECONDS + ) + credentials = context.credentials.get(process.name) + user = credentials.username if credentials else None + password = credentials.password if credentials else None + # Always the default `postgres` database — guaranteed to exist the instant `initdb` finishes, + # regardless of which baseline/world database this particular spawn is ultimately building + # toward. This call only has to prove "the engine accepts connections." + dbname = "postgres" if process.engine is ManagedEngine.POSTGRES else None + protocol = _PROTOCOL_BY_ENGINE[process.engine] + + def ready() -> bool: + if not context.prober( + protocol=protocol, host="localhost", port=port, path=None, + user=user, password=password, dbname=dbname, + ): + return False + if process.engine is ManagedEngine.RABBITMQ: + # N7, p6-review-r2: the AMQP listener probed above comes up during CORE boot; the + # management plugin's HTTP listener comes up in the PLUGIN boot step that follows — + # the same B4 race, unfixed for the one engine whose seed/sentinel/canary statements + # all travel over the listener that was never probed. Both must answer. + if not context.prober( + protocol=CapabilityProtocol.HTTP, host="localhost", + port=_rabbitmq_management_port(port), path="/api/overview", + user=user, password=password, + ): + return False + return True + + _poll_until( + ready, timeout=timeout, interval=interval, clock=clock, sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "baseline", "depends_on_timeout", + f"{process.name}: did not become ready within {timeout}s", process=process.name, + ), + ) + + +# --- N2, p6-review-r2 (BLOCKER): poll the promote-path health check, don't sample it once ------- + + +def _poll_runtime_health( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> bool: + """Shared by BOTH promote sites — `provision()`'s PREPARING->READY promotion and `reset()`'s + post-sentinel probe — which used to call `probe_runtime_health` exactly once, immediately, + with no wait. `spawn_world` only waits on `depends_on` EDGES (round-1's own M4 note): nobody + depends on the DAG's TERMINAL process, so nothing ever waited for it. A bundle whose + readiness-bearing process is terminal (§2a's own documented `vapi`/`retell`-with-a-backend + shape) was marked `UNHEALTHY` on every provision and every reset regardless of how quickly it + actually came up, driving §5.4's `world_pool_exhausted` against a healthy environment. + + Same `max(declared timeout, default)` / `min(declared interval, default)` shape `_wait_for_ + store_ready` already established for the identical class of race, applied across every probe + the runtime's OWN endpoints declare (`manifest.readiness`, the same set `probe_runtime_health` + itself iterates). Exhaustion returns `False` — never raises: this feeds a state DECISION + (`READY` vs `UNHEALTHY`), not a provisioning failure the caller must escalate. + """ + timeout = max( + (probe.timeout_seconds for probe in manifest.readiness), + default=_DEFAULT_STORE_READY_TIMEOUT_SECONDS, + ) + interval = min( + (probe.interval_seconds for probe in manifest.readiness), + default=_DEFAULT_STORE_READY_INTERVAL_SECONDS, + ) + deadline = clock() + timeout + while True: + if probe_runtime_health(manifest, runtime, prober=prober): + return True + if clock() >= deadline: + return False + sleep(interval) + + +# --- §5.3 build output (`build.json`) --------------------------------------------------------- + + +@dataclass(frozen=True) +class StoreBaselineRecord: + """One store's sealed identity, as §5.3 wants it recorded on the build output. + `baseline_reference`: `template_database` -> the sealed template database's own name; + `datadir_copy` -> the baseline snapshot directory world-clone/reset copy from; + `empty` -> `""` (§5.3: "no-op capture" — there is nothing to seal). `row_counts` is populated + for postgres stores only — `HostedWorld` (the only consumer) is postgres-only.""" + + capability: str + process_name: str + engine: ManagedEngine + strategy: BaselineStrategy + inputs_digest: str + baseline_reference: str + row_counts: dict[str, int] + + def to_json(self) -> dict[str, JsonValue]: + return { + "capability": self.capability, + "process_name": self.process_name, + "engine": self.engine.value, + "strategy": self.strategy.value, + "inputs_digest": self.inputs_digest, + "baseline_reference": self.baseline_reference, + "row_counts": dict(self.row_counts), + } + + +@dataclass +class BuildOutput: + """§5.3's `build.json` artifact: "record `inputs_digest` + achieved-baseline reference on the + build output." `conformance`/`conformance_reason` start unset; the conformance gate fills them + in and the caller re-writes the file — §4's own "record pass/fail on the build output." + `requested_parallelism`/`effective_parallelism`/`degrade_reason` (m1, p6-review-r1) are what + P7's own `parallelism_degraded` event payload needs (`{requested, effective, reason}`) for + EITHER degrade cause — `PortPlan.degraded_reason` (`fixed_port`) used to be computed and never + read by this provider; only the conformance gate's own reason ever reached `build.json`. + """ + + bundle_digest: str + stores: list[StoreBaselineRecord] + conformance: bool | None = None + conformance_reason: str | None = None + requested_parallelism: int | None = None + effective_parallelism: int | None = None + degrade_reason: str | None = None + + def to_json(self) -> dict[str, JsonValue]: + return { + "bundle_digest": self.bundle_digest, + "stores": [record.to_json() for record in self.stores], + "conformance": self.conformance, + "conformance_reason": self.conformance_reason, + "requested_parallelism": self.requested_parallelism, + "effective_parallelism": self.effective_parallelism, + "degrade_reason": self.degrade_reason, + } + + +def write_build_output(work_directory: Path, build_output: BuildOutput) -> Path: + artifacts_dir = work_directory / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + target = artifacts_dir / "build.json" + target.write_text( + json.dumps(build_output.to_json(), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return target + + +# --- §5 step 3: baseline freeze ----------------------------------------------------------------- + + +@dataclass +class FreezeResult: + """`freeze_baseline`'s full output: the `BuildOutput` §5.3 wants written to `build.json`, plus + the live handles of every `template_database` engine it just sealed — that engine stays + running for the rest of the job (§2b: job-shared), so the caller (`ProcessRuntimeProvider`) + must carry its handle forward into every world rather than re-spawning it.""" + + build_output: BuildOutput + job_shared_handles: dict[str, SpawnedWorldProcess] + + +def freeze_baseline( + manifest: EnvironmentBundleV2, *, bundle_digest: str, context: SpawnContext, +) -> FreezeResult: + """§5 step 3 / §2c: seed each managed store once, then seal it per baseline strategy — the + FIRST seal; every later world clone/reset reuses what this produces (`_seal_world_store`). + Runs once per job, before any world exists — the caller is responsible for calling this + exactly once, same convention as `build_process_trees`. `context.bundle_dir` must be set (the + bundle's own directory, for locating `migrations`/`seed_files` on disk) — the one place in + this module a caller must supply it, since seeding is the one place they matter. + """ + if context.bundle_dir is None: + raise ProcessRuntimeError( + "baseline", _INTERNAL_INVARIANT_VIOLATED, "SpawnContext.bundle_dir is required to seed", + ) + store_by_process = _store_by_process_name(manifest) + records: list[StoreBaselineRecord] = [] + job_shared_handles: dict[str, SpawnedWorldProcess] = {} + try: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + store = store_by_process.get(process.name) + if store is None: + continue # §2b default: no store entry -> per-world, nothing to freeze here. + record, handle = _freeze_one_store(manifest, process, store, context=context) + records.append(record) + if handle is not None: + job_shared_handles[process.name] = handle + except BaseException: + # N4, p6-review-r2 (MAJOR): a raise partway through leaves every EARLIER store's own + # job-shared engine live and unreferenced anywhere — `self._job_shared_handles` is never + # assigned on a raise (only after this function RETURNS), so nothing can terminate it and + # nothing frees the port it holds for a same-bundle retry. Terminated here rather than + # left running: the whole attempt has already failed (`ProcessRuntimeProvider` resets the + # job identity on this same raise — N3), so nothing downstream can resume these handles. + for handle in job_shared_handles.values(): + _terminate_and_wait(handle.handle) + raise + return FreezeResult( + build_output=BuildOutput(bundle_digest=bundle_digest, stores=records), + job_shared_handles=job_shared_handles, + ) + + +def _freeze_one_store( + manifest: EnvironmentBundleV2, process: ManagedProcess, store: StoreEntry, *, + context: SpawnContext, +) -> tuple[StoreBaselineRecord, SpawnedWorldProcess | None]: + strategy = store.baseline.strategy + if strategy is BaselineStrategy.EMPTY: + # §5.3: "no-op capture" — nothing seeded, nothing sealed here; `_seal_world_store`'s + # fallback branch (re)establishes this store's declared state on every clone/reset instead. + return ( + StoreBaselineRecord( + capability=store.capability, process_name=process.name, engine=process.engine, + strategy=strategy, inputs_digest=store.baseline.inputs_digest, + baseline_reference="", row_counts={}, + ), + None, + ) + + port = context.port_plan.port_for(process.name, 0) # job-shared or not, world 0's slot is + # this bootstrap instance's own — a job-shared engine's port never varies by world_index + # (`PortPlan.port_for`); a `datadir_copy` engine has no real world yet to collide with. + credentials = context.credentials.get(process.name) + data_dir = managed_engine_data_dir(context.work_directory, process.name, world_index=None) + handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + try: + # B4, p6-review-r1: the engine has just been `exec`'d — the very next thing this function + # does is talk to it. + _wait_for_store_ready(manifest, process, port=port, context=context) + + baseline_dbname = ( + _template_database_name(process.name) if process.engine is ManagedEngine.POSTGRES + else "" + ) + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials( + credentials, stage="baseline", process_name=process.name + ) + _call_sql( + context.sql_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, + dbname="postgres", statement=f'CREATE DATABASE "{baseline_dbname}"', + ) + apply_store_seed( + store, engine=process.engine, bundle_dir=context.bundle_dir, port=port, + dbname=baseline_dbname, credentials=credentials, process_name=process.name, + sync_run=context.sync_run, user=handle.uid, group=handle.gid, + rabbitmq_import=context.rabbitmq_import, + ) + # m3, p6-review-r1: §2c defines the sentinel as a check "against the freshly seeded + # baseline" — checked here, before sealing, so a seed that silently produced the wrong + # state is caught at freeze (surfaced as `seed_failed` — the seed's own content did not + # produce what its own sentinel expects, a deterministic authoring fault) rather than + # first noticed at the scheduler's first `reset`, long after `build.json` already + # recorded success. + sentinel_dbname = baseline_dbname if process.engine is ManagedEngine.POSTGRES else None + if not check_sentinel( + store, engine=process.engine, host="localhost", port=port, dbname=sentinel_dbname, + credentials=credentials, sql_runner=context.sql_runner, + redis_runner=context.redis_runner, + rabbitmq_inspector=context.rabbitmq_inspector, process_name=process.name, + stage="baseline", + ): + raise ProcessRuntimeError( + "baseline", "seed_failed", + f"{process.name}: sentinel check failed against the freshly seeded baseline", + process=process.name, + ) + + row_counts: dict[str, int] = {} + baseline_reference = "" + result_handle: SpawnedWorldProcess | None = None + if strategy is BaselineStrategy.TEMPLATE_DATABASE: + if process.engine is ManagedEngine.POSTGRES: + row_counts = _measure_postgres_row_counts( + host="localhost", port=port, credentials=credentials, dbname=baseline_dbname, + sql_runner=context.sql_runner, process_name=process.name, + ) + # Sealed: marked a TEMPLATE and closed to new connections, so nothing can write into + # it after this — "the sealed post-migrate+seed datastore state" (glossary, "Baseline"). + _call_sql( + context.sql_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, + dbname="postgres", + statement=( + f'ALTER DATABASE "{baseline_dbname}" WITH IS_TEMPLATE true ' + "ALLOW_CONNECTIONS false" + ), + ) + baseline_reference = baseline_dbname + result_handle = handle # stays running — job-shared for the rest of the job. + elif strategy is BaselineStrategy.DATADIR_COPY: + if process.engine is ManagedEngine.POSTGRES: + row_counts = _measure_postgres_row_counts( + host="localhost", port=port, credentials=credentials, dbname=baseline_dbname, + sql_runner=context.sql_runner, process_name=process.name, + ) + if process.engine is ManagedEngine.REDIS: + # Q1, p6-review-r3 (MAJOR): `redis_daemon_argv` disables save points (`--save + # ""`), so a plain SIGTERM shutdown persists nothing — without an explicit + # synchronous `SAVE` first, the copied data dir below is an empty baseline. + _call_redis( + context.redis_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, command=["SAVE"], + ) + # M7, p6-review-r1: waits for the engine to actually exit before the snapshot copy + # below — a bare `terminate()` only sends SIGTERM, and postgres's smart shutdown may + # not have even started, let alone finished, by the next line. Q12, p6-review-r3: a + # longer wait for rabbitmq — its broker shutdown routinely exceeds the 5s default and + # a SIGKILL mid-mnesia-write would seal a corrupt snapshot. + _terminate_and_wait( + handle.handle, + timeout=( + _RABBITMQ_TERMINATE_WAIT_SECONDS if process.engine is ManagedEngine.RABBITMQ + else _TERMINATE_WAIT_SECONDS + ), + ) + baseline_dir = _datadir_copy_baseline_path(context.work_directory, process.name) + try: + if baseline_dir.exists(): + shutil.rmtree(baseline_dir) + (context.copy or _copytree_preserving_symlinks)(data_dir, baseline_dir) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): §4.6 — this IS the "baseline/seal copies" half of the + # mapping; sealing the very first snapshot every world clones/resets from is a + # filesystem operation, not a store-command seam B5 already typed. + raise ProcessRuntimeError( + "baseline", "store_statement_failed", + f"{process.name}: sealing the datadir_copy baseline: {exc}", + process=process.name, + ) from exc + baseline_reference = str(baseline_dir) + # `result_handle` stays `None` — every world starts its OWN engine instance from this + # snapshot (`_seal_world_store`'s `DATADIR_COPY` branch), never this bootstrap one. + + record = StoreBaselineRecord( + capability=store.capability, process_name=process.name, engine=process.engine, + strategy=strategy, inputs_digest=store.baseline.inputs_digest, + baseline_reference=baseline_reference, row_counts=row_counts, + ) + return record, result_handle + except BaseException: + # N4, p6-review-r2 (MAJOR): any raise between spawn and the successful return (a readiness + # timeout, a bad seed file, a failing freeze-time sentinel) used to leave THIS store's own + # just-spawned engine live and referenced NOWHERE — `freeze_baseline`'s own loop never + # even sees `handle` when this function raises instead of returning. Best-effort: + # `DATADIR_COPY`'s own success path has usually already terminated it by this point, and + # `_terminate_and_wait` tolerates an already-dead handle. + _terminate_and_wait(handle.handle) + raise + + +# --- §4.2/§4 world clone + reset ----------------------------------------------------------------- + + +def _seal_world_store( + manifest: EnvironmentBundleV2, + process: ManagedProcess, + store: StoreEntry | None, + record: StoreBaselineRecord | None, + *, + world_index: int, + context: SpawnContext, +) -> SpawnedWorldProcess | None: + """Drives ONE world's managed store to the sealed baseline, per §4.2 — shared by the FIRST + clone (`ProcessRuntimeProvider._ensure_world`) and every later `reset_world`; the caller is + expected to have already terminated any existing per-world handle for `process.name` before + calling this again. Returns the new per-world `SpawnedWorldProcess` for a per-world engine + (`datadir_copy`/`empty`/no-entry), or `None` for a job-shared `template_database` engine — + nothing new to spawn there, only its logical `wN` database changes. + """ + port = context.port_plan.port_for(process.name, world_index) + data_dir = managed_engine_data_dir(context.work_directory, process.name, world_index=world_index) + credentials = context.credentials.get(process.name) + strategy = store.baseline.strategy if store is not None else None + + if strategy is BaselineStrategy.TEMPLATE_DATABASE: + if record is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process.name}: template_database has no frozen baseline record", + process=process.name, + ) + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + # No spawn on this branch (the job-shared engine is already running, and already went + # through B4's wait once at freeze/first-clone time) — nothing new to wait on here. + _reset_template_database( + host="localhost", port=port, credentials=credentials, world_db=f"w{world_index}", + template_db=record.baseline_reference, sql_runner=context.sql_runner, + process_name=process.name, + ) + return None + + if strategy is BaselineStrategy.DATADIR_COPY: + # R1, p6-review-r4 (known defect, disclosed rather than redesigned this phase): for + # rabbitmq specifically, this branch restores the mnesia DIRECTORY but not a schema + # rabbitmq will actually adopt — mnesia data is node-name-bound and per-world node names + # must stay distinct for epmd (see `rabbitmq_daemon_env`'s docstring), so a rabbitmq + # `datadir_copy` baseline boots correctly in world 0 only. Recorded follow-up: a + # definitions-export restore via `default_rabbitmq_definitions_importer`. + if record is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process.name}: datadir_copy has no frozen baseline record", process=process.name, + ) + try: + if data_dir.exists(): + shutil.rmtree(data_dir) + # No pre-`mkdir` here (unlike the fallback branch below) — `copytree`-shaped `copy` + # creates its own destination and raises `FileExistsError` if it is already there. + (context.copy or _copytree_preserving_symlinks)( + Path(record.baseline_reference), data_dir + ) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): restoring a world's engine from its sealed baseline + # snapshot is itself an infrastructure operation (§4.6) — the "baseline/seal copies" + # half of the mapping, same class `_freeze_one_store`'s own snapshot copy got. + raise ProcessRuntimeError( + "reset", "store_statement_failed", + f"{process.name}: restoring the datadir_copy baseline: {exc}", + process=process.name, + ) from exc + new_handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + # B4, p6-review-r1: fresh spawn from a just-restored data directory — the next line talks + # to it immediately. + _wait_for_store_ready(manifest, process, port=port, context=context) + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + world_db = f"w{world_index}" + baseline_dbname = _template_database_name(process.name) + # §2b: "under `datadir_copy` the provisioner configures each per-world engine's + # database to `w` as well — one rule, both strategies." The copied dir's own + # database still carries `_freeze_one_store`'s bootstrap name; renamed here, once per + # world, never touching the pristine snapshot every OTHER world/reset still copies from. + _call_sql( + context.sql_runner, stage="reset", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'ALTER DATABASE "{baseline_dbname}" RENAME TO "{world_db}"', + ) + return new_handle + + # `empty` strategy, or no `seed.stores` entry at all: no baseline snapshot exists to copy back + # in (`freeze_baseline` never ran for it — §5.3's "no-op capture" / §2b's per-world default), + # so this always starts the engine over a WIPED, empty data directory — unconditionally, since + # a bare restart alone cannot be trusted to flush every engine (rabbitmq persists to disk + # regardless of redis's own `--save ""`). An explicit `empty` store then re-applies its own + # seed_files on top — every clone/reset is where its declared initial state actually gets + # (re)established, since freeze deliberately skipped it. + try: + if data_dir.exists(): + shutil.rmtree(data_dir) + data_dir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + # N9, p6-review-r2 (MAJOR): a bare wipe-and-recreate of this process/data-dir setup, same + # class as `spawn_managed_process`'s own boundary. + raise ProcessRuntimeError( + "reset", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", + process=process.name, + ) from exc + handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + # B4, p6-review-r1: fresh spawn over a wiped data directory — the CREATE DATABASE/apply_ + # store_seed below talk to it immediately. + _wait_for_store_ready(manifest, process, port=port, context=context) + if store is not None and strategy is BaselineStrategy.EMPTY: + dbname = f"w{world_index}" if process.engine is ManagedEngine.POSTGRES else "" + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + _call_sql( + context.sql_runner, stage="reset", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'CREATE DATABASE "{dbname}"', + ) + if context.bundle_dir is None: + # Same invariant `freeze_baseline` guards explicitly — a silent `Path()` (cwd) fallback + # here would resolve `seed_files` against the wrong directory instead of failing typed. + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + "SpawnContext.bundle_dir is required to re-seed an empty-strategy store", + process=process.name, + ) + apply_store_seed( + store, engine=process.engine, bundle_dir=context.bundle_dir, port=port, + dbname=dbname, credentials=credentials, process_name=process.name, + sync_run=context.sync_run, user=handle.uid, group=handle.gid, + rabbitmq_import=context.rabbitmq_import, + ) + return handle + + +def _reset_template_database( + *, host: str, port: int, credentials: EngineCredentials, world_db: str, template_db: str, + sql_runner: SqlRunner, process_name: str, +) -> None: + """§4.2 `template_database`: "drop + recreate the world's logical DB from the template." The + admin connection targets `postgres` (never `world_db` itself or the template — postgres + forbids `DROP`/`CREATE DATABASE` from inside the database being touched). Terminates other + backends on `world_db` first — a connection lingering from that world's own just-terminated + `source` processes is exactly what would otherwise block the `DROP`. + """ + admin = dict( + host=host, port=port, user=credentials.username, password=credentials.password, + dbname="postgres", + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, + statement=( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{world_db}' AND pid <> pg_backend_pid()" + ), + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, statement=f'DROP DATABASE IF EXISTS "{world_db}"', + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, statement=f'CREATE DATABASE "{world_db}" TEMPLATE "{template_db}"', + ) + + +def _clone_or_reset_world( + manifest: EnvironmentBundleV2, + world_index: int, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + existing_handles: dict[str, SpawnedWorldProcess], +) -> WorldSpawnResult: + """Shared by `ProcessRuntimeProvider._ensure_world` (first creation / sick-world replace) and + `reset_world` (mid-job restore) — both are "terminate this world's own per-world handles, + reseal each managed store from the baseline, respawn `source` processes"; they differ only in + what the caller does afterward (a fresh/replaced world is left `preparing`, judged later by + `healthy()`'s declared readiness probes; a reset world is sentinel-checked immediately, per + §4.2, and marked `unhealthy` on failure). + """ + # N12, p6-review-r2 (MINOR): reversed — `existing_handles`' insertion order is `spawn_world`'s + # own `_topological_order` (dependencies first); terminating it forward sends SIGTERM to this + # world's own postgres while its `tools-api`/`agent` may still hold connections, guaranteeing + # the full escalation wait every reset. A dict preserves insertion order, so `reversed()` here + # IS reverse-topological order without recomputing it. + for name, handle in reversed(list(existing_handles.items())): + if name not in job_shared_handles: + # M7, p6-review-r1: waits for real exit before `_seal_world_store` below `rmtree`s + # this same process's data directory and rebinds its port — a bare `terminate()` + # racing that `rmtree` is exactly `EADDRINUSE` / "remove a live server's data dir". + _terminate_and_wait( + handle.handle, prefer_interrupt=_prefers_interrupt(manifest, name), + ) + + store_by_process = _store_by_process_name(manifest) + baseline_by_process = {record.process_name: record for record in baseline.stores} + new_handles: dict[str, SpawnedWorldProcess] = dict(job_shared_handles) + try: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + # `_seal_world_store` runs even for an already-running job-shared engine + # (`process.name` already in `new_handles`, seeded from `job_shared_handles` above) + # — its own logical `wN` database still needs resetting every time, even though the + # shared ENGINE PROCESS itself does not; `template_database`'s branch returns `None`, + # so `new_handles` is left holding the pre-seeded shared handle untouched, exactly as + # intended. + store = store_by_process.get(process.name) + record = baseline_by_process.get(process.name) + sealed = _seal_world_store( + manifest, process, store, record, world_index=world_index, context=context, + ) + if sealed is not None: + new_handles[process.name] = sealed + except BaseException as exc: + # N4, p6-review-r2 (MAJOR): a raise partway through this loop (a bad seed_files entry on + # a LATER process, `_wait_for_store_ready` timing out) used to drop every EARLIER + # process's freshly-(re)sealed handle on the floor — nothing holds it, so `close()`'s own + # `rmtree` of this world's data directory next runs against a still-live engine. Attached + # to the exception (never terminated here) so the caller — which alone knows whether this + # is a first clone with nothing to lose, or a reset it may still want to retry — can merge + # it into its own live-handle bookkeeping before deciding what to do next. + exc.partial_handles = new_handles # type: ignore[attr-defined] + raise + + # `spawn_world` sets its own (more complete — it starts from `new_handles` and accumulates + # further) `partial_handles` on a raise, so no extra wrapping is needed here for that case. + return spawn_world(manifest, world_index=world_index, context=context, shared_handles=new_handles) + + +def _check_all_sentinels( + manifest: EnvironmentBundleV2, world_index: int, *, context: SpawnContext, +) -> bool: + """§4.2: "after every reset the store's sentinel must pass" — every declared store, not just + the first; a single failing sentinel fails the whole check (short-circuits nothing, so every + store is always attempted — useful diagnostics beat an early return here).""" + store_by_process = _store_by_process_name(manifest) + ok = True + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + store = store_by_process.get(process.name) + if store is None: + continue + port = context.port_plan.port_for(process.name, world_index) + dbname = f"w{world_index}" if process.engine is ManagedEngine.POSTGRES else None + passed = check_sentinel( + store, engine=process.engine, host="localhost", port=port, dbname=dbname, + credentials=context.credentials.get(process.name), sql_runner=context.sql_runner, + redis_runner=context.redis_runner, rabbitmq_inspector=context.rabbitmq_inspector, + process_name=process.name, stage="reset", + ) + ok = ok and passed + return ok + + +def reset_world( + manifest: EnvironmentBundleV2, + world_index: int, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + existing_handles: dict[str, SpawnedWorldProcess], +) -> tuple[dict[str, SpawnedWorldProcess], bool]: + """§4.2's per-world reset, exactly — returns the world's refreshed handle map and whether + every declared store's sentinel passed afterward. NEVER raises for a sentinel failure: §4.2's + own words, "a sentinel failure marks the world unhealthy," is the caller's job to act on, not + this function's to escalate. A genuine provisioning failure below this (a `ProcessRuntimeError` + from spawn/build) still raises — that is not what "sentinel failure" covers. + """ + result = _clone_or_reset_world( + manifest, world_index, context=context, baseline=baseline, + job_shared_handles=job_shared_handles, existing_handles=existing_handles, + ) + ok = _check_all_sentinels(manifest, world_index, context=context) + return result.handles, ok + + +# --- §4 conformance gate ------------------------------------------------------------------------- + +_CONFORMANCE_CANARY_NAME = "_alk_conformance" # §2c reserved name; mirrors process_preflight.py's +# own `_RESERVED_NAME` and `world/handle.py`'s `CONFORMANCE_TABLE` — redeclared locally rather +# than imported, matching the existing split of that same constant across those two modules. +_CONFORMANCE_MARKER = "alk-conformance-canary" + +_CANARY_PROTOCOL_PREFERENCE = ( + CapabilityProtocol.POSTGRES, CapabilityProtocol.REDIS, CapabilityProtocol.AMQP, +) + + +def _first_canary_store(manifest: EnvironmentBundleV2) -> StoreEntry | None: + """§4: "the first store by protocol preference postgres > redis > rabbitmq." `no_sql_store` + (§2e item 6) guarantees a `kind: process` bundle always has a postgres store, so `None` is + reachable only for the vacuous zero-store case §4 itself names (never actually a `kind: + process` bundle in practice).""" + if manifest.seed is None: + return None + by_protocol: dict[CapabilityProtocol, StoreEntry] = {} + for store in manifest.seed.stores: + capability = manifest.capabilities[store.capability] + by_protocol.setdefault(capability.protocol, store) + for protocol in _CANARY_PROTOCOL_PREFERENCE: + if protocol in by_protocol: + return by_protocol[protocol] + return None + + +def _engine_for_store(manifest: EnvironmentBundleV2, store: StoreEntry) -> ManagedEngine: + capability = manifest.capabilities[store.capability] + processes_by_name = {process.name: process for process in manifest.processes} + process = processes_by_name[capability.service] + if not isinstance(process, ManagedProcess): + raise ProcessRuntimeError( + "conformance", _INTERNAL_INVARIANT_VIOLATED, + f"{store.capability}: backing service is not a managed engine; bundle_v2's " + "store_service_not_managed should have rejected this bundle", + ) + return process.engine + + +def _run_canary_probe( + manifest: EnvironmentBundleV2, store: StoreEntry, engine: ManagedEngine, *, context: SpawnContext, +) -> bool: + """Creates the reserved object in world 0, then asserts it is NOT visible in world 1 — proving + the two worlds are really isolated from each other, not just reachable on different ports.""" + capability = manifest.capabilities[store.capability] + process_name = capability.service + credentials = context.credentials.get(process_name) + port0 = context.port_plan.port_for(process_name, 0) + port1 = context.port_plan.port_for(process_name, 1) + + if engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, user=credentials.username, password=credentials.password, + dbname="w0", + statement=( + f'CREATE TABLE "{_CONFORMANCE_CANARY_NAME}" (marker text); ' + f"INSERT INTO \"{_CONFORMANCE_CANARY_NAME}\" VALUES ('{_CONFORMANCE_MARKER}')" + ), + ) + rows = _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port1, user=credentials.username, password=credentials.password, + dbname="w1", statement=f"SELECT to_regclass('{_CONFORMANCE_CANARY_NAME}') IS NOT NULL", + read_only=True, # N8, p6-review-r2. + ) + return not (rows and rows[0] and rows[0][0]) + if engine is ManagedEngine.REDIS: + _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, + command=["SET", _CONFORMANCE_CANARY_NAME, _CONFORMANCE_MARKER], + ) + value = _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port1, command=["GET", _CONFORMANCE_CANARY_NAME], + ) + return value is None + if engine is ManagedEngine.RABBITMQ: + # rabbitmq is never job-shared (§2b: not in `_ENGINE_STRATEGIES`'s `template_database` + # set) — world 0 and world 1 are already separate broker processes on separate ports, so + # there is no shared state left to leak between them. Declaring the queue in world 0 and + # confirming world 1's own broker never saw it is what is left to prove — m8, p6-review- + # r1: previously only ever INSPECTED (a read), so world 1's check compared against a + # queue that had never been created anywhere, a vacuous pass regardless of isolation. + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + _call_rabbitmq_action( + context.rabbitmq_declare, stage="conformance", process_name=process_name, + action="declare", + host="localhost", port=port0, credentials=credentials, + queue=_CONFORMANCE_CANARY_NAME, message=_CONFORMANCE_MARKER, + ) + depth = _call_rabbitmq_with_retry( + context.rabbitmq_inspector, stage="conformance", process_name=process_name, + host="localhost", port=port1, credentials=credentials, queue=_CONFORMANCE_CANARY_NAME, + accept=lambda value: value in (0, None), + ) + # N15, p6-review-r2 (MINOR): no explicit delete here anymore. `run_conformance_gate`'s own + # `reset_world` calls for BOTH worlds run right after this returns, and world 0's rabbitmq + # reset (`datadir_copy`, its only legal strategy — §2b) wipes and restarts it from the + # pristine baseline snapshot regardless — the same mechanism postgres/redis already rely + # on for their own canary cleanup. Deleting HERE used to make `_verify_canary_absent`'s + # LATER read of this same queue vacuous: it was already gone from an earlier, unrelated + # step, proving nothing about whether the reset itself actually worked. A failed probe + # (isolation broken) still returns here without a reset ever running — the reserved name + # keeps that residual harmless (never collides with customer content). + return depth in (0, None) + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + +def _verify_canary_absent( + manifest: EnvironmentBundleV2, store: StoreEntry, engine: ManagedEngine, *, context: SpawnContext, +) -> bool: + """§4: "assert it is gone" — checked against world 0's OWN namespace after both worlds reset, + defense in depth against a reset that silently failed to actually seal a fresh baseline.""" + capability = manifest.capabilities[store.capability] + process_name = capability.service + credentials = context.credentials.get(process_name) + port0 = context.port_plan.port_for(process_name, 0) + + if engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + rows = _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, user=credentials.username, password=credentials.password, + dbname="w0", statement=f"SELECT to_regclass('{_CONFORMANCE_CANARY_NAME}') IS NOT NULL", + read_only=True, # N8, p6-review-r2. + ) + return not (rows and rows[0] and rows[0][0]) + if engine is ManagedEngine.REDIS: + value = _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, command=["GET", _CONFORMANCE_CANARY_NAME], + ) + return value is None + if engine is ManagedEngine.RABBITMQ: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + depth = _call_rabbitmq_with_retry( + context.rabbitmq_inspector, stage="conformance", process_name=process_name, + host="localhost", port=port0, credentials=credentials, queue=_CONFORMANCE_CANARY_NAME, + accept=lambda value: value in (0, None), # N15, p6-review-r2. + ) + return depth in (0, None) + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + +def run_conformance_gate( + manifest: EnvironmentBundleV2, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + world_handles: dict[int, dict[str, SpawnedWorldProcess]], +) -> tuple[bool, str | None]: + """§4's 2-world canary, run once per attempt after baseline freeze: create the reserved + `_alk_conformance` object in world 0, assert it is invisible in world 1, `reset` both, assert + it is gone + every sentinel passes both worlds. TRULY never raises (M3, p6-review-r1) — every + gate-procedure check returns `(False, "conformance_gate_failed")`, and a `ProcessRuntimeError` + RAISED below this (B5's typed store-seam failures, `reset_world`'s own spawn/reseal path, a + missing-credentials invariant) is now caught and degrades the SAME way rather than failing the + whole job: §4's own words are "Fail -> effective parallelism 1 ... Loud, never silent," which + governs the GATE'S RESULT, not just a False return from its own checks — a world that has not + finished booting when the canary dials it (B4 narrows but does not eliminate this race) is an + isolation-boundary problem exactly like a failed canary, not a reason to fail the job. Requires + worlds 0 and 1 to already exist in `world_handles` — the caller (`ProcessRuntimeProvider`) + provisions a throwaway 2-world pair before calling this, then reconciles down to 1 on a + `False` return, or up to the job's real `W` on `True`. + """ + store = _first_canary_store(manifest) + if store is None: + return True, None # §2e's own no_sql_store guarantee means this never actually happens. + engine = _engine_for_store(manifest, store) + + world_index: int | None = None + try: + if not _run_canary_probe(manifest, store, engine, context=context): + return False, "conformance_gate_failed" + + for world_index in (0, 1): + handles, sentinel_ok = reset_world( + manifest, world_index, context=context, baseline=baseline, + job_shared_handles=job_shared_handles, + existing_handles=world_handles.get(world_index, {}), + ) + world_handles[world_index] = handles + if not sentinel_ok: + return False, "conformance_gate_failed" + + if not _verify_canary_absent(manifest, store, engine, context=context): + return False, "conformance_gate_failed" + return True, None + except (ProcessRuntimeError, OSError, shutil.Error) as exc: + # N9, p6-review-r2: widened past `ProcessRuntimeError` alone — `reset_world` below this + # runs `rmtree`/`copytree`/`chown`/`chmod` (`_seal_world_store`'s own work), and a bare + # filesystem fault there used to escape this "TRULY never raises" (M3) gate untyped. + # N4, p6-review-r2: a raise from `reset_world` (via `_clone_or_reset_world`) mid-loop + # carries whatever THIS world's own partial reseal produced (`exc.partial_handles`) — + # merged here so a live engine the gate's own reset just spawned is not left unreferenced + # in `world_handles` the moment the gate degrades instead of propagating the failure. + partial = getattr(exc, "partial_handles", None) + if partial is not None and world_index is not None: + world_handles[world_index] = partial + # The cause cannot travel in the returned `reason` string — that value is §5's own closed + # `parallelism_degraded.reason` vocabulary (`conformance_gate_failed` | `fixed_port`) — so + # "loud" means logged here, not encoded in the return. + if isinstance(exc, ProcessRuntimeError): + logger.error( + "conformance gate raised %s/%s: %s degrading to effective parallelism 1 instead " + "of failing the job", exc.stage, exc.code, exc, + ) + else: + logger.error( + "conformance gate raised %s: %s degrading to effective parallelism 1 instead of " + "failing the job", type(exc).__name__, exc, + ) + return False, "conformance_gate_failed" + + +# --- §0.3/§4 rule 1 secrets lifetime (B3, p6-review-r1) --------------------------------------- + + +def _read_job_secret_purposes(work_directory: Path) -> dict[str, str]: + """§0.2/§4.1: `/work/job.json` is the provisioner's own configuration source — `agent. + secret_refs` (§1: `{alias: {manager, key, version, purpose}}`) is where each alias's REAL + purpose comes from. N10, p6-review-r2 (MAJOR): every alias used to be relabelled `target_ + provider` unconditionally in `_load_and_delete_secrets`, which silently defeats `select_ + process_secrets`'s own `SOURCE_CHECKOUT` exclusion (F13) the moment a `source_checkout` alias + ever reaches `secrets.json` — the guest must not depend on the gateway alone never putting one + there, which is exactly the promise F13's own docstring says it will not depend on. Q11, + p6-review-r3: the two raises below use `spawn_failed` for a malformed `job.json`, which is + also a vocabulary stretch — §2f's domain rule for it is "infrastructure if a managed engine, + `agent` if source," neither of which is what a config-read fault actually is; picked as the + closest §4.6 code available, same reasoning as `store_statement_failed`'s stretch elsewhere. + """ + job_json_path = work_directory / "job.json" + if not job_json_path.exists(): + # A real gap on the hosted path (§0.2 guarantees the file), not a local-lane default to + # paper over silently — the caller still proceeds (a constructor-supplied `secret_purpose_ + # map`, or the local/test lane, never needs this file at all), but every alias in + # `secrets.json` then has no purpose to match and is correctly dropped below, which would + # otherwise look like a silent injection failure three layers down with nothing in the + # log explaining why. + logger.warning( + "secrets: %s is absent; every alias in secrets.json will be dropped (no purpose to " + "match) unless a secret_purpose_map was supplied", job_json_path, + ) + return {} + try: + raw = json.loads(job_json_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProcessRuntimeError( + "secrets", "spawn_failed", f"{job_json_path}: unreadable or not valid JSON: {exc}", + ) from exc + if not isinstance(raw, dict): + raise ProcessRuntimeError( + "secrets", "spawn_failed", + f"{job_json_path}: expected a JSON object, got {type(raw).__name__}", + ) + agent = raw.get("agent") + refs = agent.get("secret_refs") if isinstance(agent, dict) else None + if not isinstance(refs, dict): + return {} + return { + str(alias): ref["purpose"] + for alias, ref in refs.items() + if isinstance(ref, dict) and isinstance(ref.get("purpose"), str) + } + + +def _load_and_delete_secrets( + secrets_path: Path, *, work_directory: Path, secret_purpose_map: dict[str, str] | None = None, +) -> tuple[dict[str, str], dict[str, str]]: + """§0 step 3 / §4 rule 1's lifetime rule: "the provisioner loads this file into memory at + startup and deletes it immediately after loading, BEFORE ANY CUSTOMER PROCESS STARTS. The + in-memory map lives for the whole job — `reset` restarts and `provision` reconciliations + re-inject from memory." + + N10, p6-review-r2 (MAJOR): each alias's purpose comes from `job.json`'s own `agent. + secret_refs` (`_read_job_secret_purposes`) — NOT invented as `target_provider` for every + alias, which used to retire `select_process_secrets`'s `SOURCE_CHECKOUT` exclusion the moment + a `source_checkout` alias ever reached this file. `secret_purpose_map`, when given, overrides + the job.json read entirely (the local/test lane's own shape — no `/work/job.json` on a dev + box). An alias in `secrets.json` with no matching ref anywhere is dropped, not injected under + a guessed purpose, and logged — `select_process_secrets` naturally never matches an alias + absent from the purposes map, so nothing further is needed to enforce the drop. + """ + if not secrets_path.exists(): + values: dict[str, str] = {} + else: + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProcessRuntimeError( + "secrets", "spawn_failed", f"{secrets_path}: unreadable or not valid JSON: {exc}", + ) from exc + secrets_path.unlink(missing_ok=True) + if not isinstance(raw, dict): + raise ProcessRuntimeError( + "secrets", "spawn_failed", + f"{secrets_path}: expected a JSON object of alias -> value, got " + f"{type(raw).__name__}", + ) + values = {str(alias): str(value) for alias, value in raw.items()} + + all_purposes = ( + dict(secret_purpose_map) if secret_purpose_map is not None + else _read_job_secret_purposes(work_directory) + ) + dropped = sorted(alias for alias in values if alias not in all_purposes) + if dropped: + logger.warning( + "secrets: %d alias(es) with no matching agent.secret_refs entry will never be " + "injected into any process: %s", len(dropped), dropped, + ) + purposes = {alias: all_purposes[alias] for alias in values if alias in all_purposes} + return values, purposes + + +# --- §4 provision / close: the stateful RuntimeProvider adapter ----------------------------------- + + +class ProcessRuntimeProvider: + """The §4 `RuntimeProvider`, structurally: a stateful adapter around this module's pure + functions (`spawn_world`, `freeze_baseline`, `reset_world`, `run_conformance_gate`, ...), + holding what a §3-shaped `EnvironmentRuntime` cannot carry itself — live process handles, the + job's generated credentials, and the sealed baseline. `runtime.py`'s own `RuntimeProvider` + Protocol is untouched (P5's docstring: "a later phase wires a Protocol-conforming, stateful + adapter around the pure functions below") — wiring the existing local-SDK callers onto this + shape is the refactor-sized "Implementation delta" §4 itself calls out, a separate change from + this one; `provision`/`reset`/`close` below match §4's method shapes (not `runtime.py`'s older + v1 ones — no `provider` field, `provision` takes `instances` and returns a list, `close` takes + no `runtime` argument), since those are what this phase actually implements. + + Idempotency (§4 rule 1, "idempotent for the job identity") holds for repeated calls against the + SAME instance — the normal in-job shape: the entrypoint constructs one provider per job and + calls `provision` once, then `reset`/`provision` again for sick-world recovery, all against + that one long-lived object for the guest process's whole life. Resuming after the GUEST + PROCESS ITSELF restarts (a fresh `hosted_entrypoint` invocation after a crash) would need + filesystem-observable state beyond what this class attempts — `write_build_output`'s own + `inputs_digest` reuse (§2c: "the provisioner records it... as the baseline identity for + attempt-retry reuse") is the one piece of that this module produces; resurrecting LIVE process + handles across a process boundary is the entrypoint's own concern, out of this phase's file + scope. + + n2, p6-review-r2: this class takes no lock over its own mutable state (`_world_handles`, + `_runtimes`, `_context`, `_secrets_loaded`, ...) behind the `asyncio.to_thread` calls below — + that is deliberate, not an oversight. v1.12 §4.5b makes the port NON-REENTRANT and puts + serialization on the SCHEDULER, never the provider: every `provision`/`reset`/`healthy`/ + `close` call for one job must be serialized by the caller — `healthy` demotes state, so it + writes and is in the set too (Q9, p6-review-r3). Do not add a lock here on the assumption one + is missing; add it at the call site instead. + """ + + name = "hosted-process" + + def __init__( + self, + *, + runner: ProcessRunner = default_process_runner, + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + prober: CapabilityProber = default_capability_prober, + sql_runner: SqlRunner = default_sql_runner, + redis_runner: RedisCommandRunner = default_redis_command_runner, + rabbitmq_inspector: RabbitmqQueueInspector = default_rabbitmq_queue_inspector, + rabbitmq_declare: RabbitmqQueueDeclarer = default_rabbitmq_queue_declare_and_publish, + rabbitmq_delete: RabbitmqQueueDeleter = default_rabbitmq_queue_delete, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, + copy: Callable[[Path, Path], None] | None = None, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + chown: Callable[[Path, int, int], None] = _default_chown, + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS, + token: Callable[[], str] | None = None, + secrets_path: Path = Path("/run/futureagi/secrets.json"), + close_wait_timeout_seconds: float = _TERMINATE_WAIT_SECONDS, + secret_purpose_map: dict[str, str] | None = None, + ) -> None: + self._runner = runner + self._sync_run = sync_run + self._prober = prober + self._sql_runner = sql_runner + self._redis_runner = redis_runner + self._rabbitmq_inspector = rabbitmq_inspector + self._rabbitmq_declare = rabbitmq_declare + self._rabbitmq_delete = rabbitmq_delete + self._rabbitmq_import = rabbitmq_import + self._copy = copy + self._user_resolver = user_resolver + self._chown = chown + self._build_step_timeout_seconds = build_step_timeout_seconds + self._token = token + self._secrets_path = secrets_path + # N12, p6-review-r2: `close()`'s own bound on the per-handle wait/kill escalation — §0.7's + # 120s flush window is shared by every handle `_teardown_processes_and_directories` tears + # down, so a caller close to that deadline can pass something tighter than the module + # default without touching every OTHER termination call site's own timeout. + self._close_wait_timeout_seconds = close_wait_timeout_seconds + # N10, p6-review-r2: overrides the `/work/job.json` read entirely when supplied — the + # local/test lane's own shape (no job.json on a dev box); a hosted caller leaves this + # `None` and `_load_and_delete_secrets` reads `agent.secret_refs` itself. + self._secret_purpose_map = secret_purpose_map + + self._manifest: EnvironmentBundleV2 | None = None + self._bundle_digest: str | None = None + self._context: SpawnContext | None = None + self._build_output: BuildOutput | None = None + self._job_shared_handles: dict[str, SpawnedWorldProcess] = {} + self._world_handles: dict[int, dict[str, SpawnedWorldProcess]] = {} + self._runtimes: dict[int, EnvironmentRuntime] = {} + self._conformance_checked = False + # B3, p6-review-r1: loaded once (`_load_and_delete_secrets`), on the FIRST provision call + # for a job identity — `_secrets_loaded` latches so neither a later reconcile call nor a + # bundle-digest rebuild (M6) ever tries to re-read a file `close()`/the load itself has + # already deleted; §0.3's "re-inject from memory" is exactly these two dicts surviving on + # `self` across every `provision`/`reset` call for the object's whole life. + self._secret_values: dict[str, str] = {} + self._secret_purposes: dict[str, str] = {} + self._secrets_loaded = False + + async def provision( + self, + bundle: EnvironmentBundleV2, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, + contract: Any | None = None, # accepted for §4 shape-compatibility; not consumed here — + # evidence-seam wiring is out of this phase's scope, same as `runtime.py`'s own providers. + instances: int = 1, + require_declared_user: bool = True, + ) -> list[EnvironmentRuntime]: + import asyncio + + return await asyncio.to_thread( + self._provision_sync, bundle, source=source, bundle_dir=bundle_dir, + work_directory=work_directory, instances=instances, + require_declared_user=require_declared_user, + ) + + def _provision_sync( + self, bundle: EnvironmentBundleV2, *, source: Path, bundle_dir: Path, work_directory: Path, + instances: int, require_declared_user: bool, + ) -> list[EnvironmentRuntime]: + port_plan = plan_ports(bundle, instances=instances) + effective = port_plan.effective_instances + bundle_digest = bundle.digest + + if self._manifest is None or self._bundle_digest != bundle_digest: + if self._manifest is not None: + # M6, p6-review-r1: a bundle-digest change used to reassign this instance's own + # identity fields straight over the PREVIOUS job's still-running processes and + # still-allocated ports — §4.1's "never duplicates" broken in the one case this + # branch exists for. Full teardown first (same mechanism `close()` uses, minus + # the secrets unlink — this is a re-sealed bundle, not a new job identity, so the + # in-memory secret map must survive it). + self._teardown_processes_and_directories(work_directory) + if not self._secrets_loaded: + # B3 / §0.3, p6-review-r1: loaded and the file deleted BEFORE any customer + # process starts — done here, before `build_process_trees`/`freeze_baseline` + # below ever spawn anything. N10, p6-review-r2: purposes come from `job.json`'s + # own `agent.secret_refs` (or the constructor override), never invented. + self._secret_values, self._secret_purposes = _load_and_delete_secrets( + self._secrets_path, work_directory=work_directory, + secret_purpose_map=self._secret_purpose_map, + ) + self._secrets_loaded = True + + # First call for this job identity, or the bundle changed underneath it (§2c: a + # digest mismatch forces a rebuild) — build once, seed+freeze once, gate once. + credentials = generate_engine_credentials(bundle, token=self._token) + context = SpawnContext( + work_directory=work_directory, port_plan=port_plan, credentials=credentials, + secret_values=self._secret_values, secret_purposes=self._secret_purposes, + runner=self._runner, sync_run=self._sync_run, + prober=self._prober, copy=self._copy, user_resolver=self._user_resolver, + require_declared_user=require_declared_user, chown=self._chown, + build_step_timeout_seconds=self._build_step_timeout_seconds, + sql_runner=self._sql_runner, redis_runner=self._redis_runner, + rabbitmq_inspector=self._rabbitmq_inspector, rabbitmq_declare=self._rabbitmq_declare, + rabbitmq_delete=self._rabbitmq_delete, rabbitmq_import=self._rabbitmq_import, + bundle_dir=bundle_dir, + ) + # N3, p6-review-r2 (MAJOR): the job identity (`_manifest`/`_bundle_digest`) is + # committed only AFTER `build_process_trees`/`freeze_baseline` both succeed — + # committing it first used to leave a failed first `provision()` claiming an identity + # with no build output, so an in-process retry (the same bundle, e.g. `hosted_ + # scheduler.py`'s background `_reconcile`) took the RECONCILE branch below instead of + # retrying the build, and hit the "context/build_output unset" invariant instead of + # the real, often-retryable cause. + try: + build_process_trees(bundle, source_root=source, context=context) + freeze_result = freeze_baseline( + bundle, bundle_digest=bundle_digest, context=context + ) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): §4.6 — filesystem failures during provisioning are + # `infrastructure`. This phase's own copy-heavy work (build trees, baseline + # snapshot/seal) used to be able to raise bare here. Q11, p6-review-r3: + # `store_statement_failed` is a vocabulary stretch for a build-tree copy fault — + # the closest §2f code in a closed table with no generic `provisioner_io_failed`, + # and it lands the correct `infrastructure` domain either way. + self._manifest = None + self._bundle_digest = None + raise ProcessRuntimeError("baseline", "store_statement_failed", str(exc)) from exc + except BaseException: + self._manifest = None + self._bundle_digest = None + raise + self._manifest = bundle + self._bundle_digest = bundle_digest + self._context = context + self._build_output = freeze_result.build_output + self._job_shared_handles = freeze_result.job_shared_handles + write_build_output(work_directory, self._build_output) + self._world_handles = {} + self._runtimes = {} + self._conformance_checked = False + else: + # Same job identity, a later reconcile call — `instances`/`require_declared_user` may + # legitimately differ (a sick-world recovery re-call), and so may `work_directory`/ + # `bundle_dir` (m7, p6-review-r1: the two used to silently diverge from whatever this + # call actually passed, since only `port_plan`/`require_declared_user` were carried + # forward here). The sealed baseline never re-runs regardless (§4 rule 1). + self._context = replace( + self._context, port_plan=port_plan, require_declared_user=require_declared_user, + work_directory=work_directory, bundle_dir=bundle_dir, + ) + + context = self._context + build_output = self._build_output + if context is None or build_output is None: + # m6, p6-review-r1: both branches above set these unconditionally — a bare `assert` + # here is stripped under `python -O`, which would turn a real "this is a bug in THIS + # function" precondition into an opaque `AttributeError` a few lines down instead of + # a typed failure. + raise ProcessRuntimeError( + "provision", _INTERNAL_INVARIANT_VIOLATED, + "context/build_output unset after the first-call/reconcile branch", + ) + + requested = instances + degrade_reason = port_plan.degraded_reason # "fixed_port", or None (m1, p6-review-r1). + + if effective > 1 and not self._conformance_checked: + self._ensure_world(0) + self._ensure_world(1) + passed, reason = run_conformance_gate( + bundle, context=context, baseline=build_output, + job_shared_handles=self._job_shared_handles, world_handles=self._world_handles, + ) + build_output.conformance = passed + build_output.conformance_reason = reason + self._conformance_checked = True + if not passed: + effective = 1 + degrade_reason = reason + elif self._conformance_checked and build_output.conformance is False: + # A degrade decided by an EARLIER call must keep holding on every later reconcile call + # too — `effective` above is freshly recomputed from `port_plan` each time and knows + # nothing about a gate result from a call that already happened. + effective = 1 + degrade_reason = "conformance_gate_failed" + + build_output.requested_parallelism = requested + build_output.effective_parallelism = effective + build_output.degrade_reason = degrade_reason + write_build_output(work_directory, build_output) + + # Reconcile down first — a prior call may have over-provisioned (the canary's own 2-world + # pair, before a gate failure dropped `effective` to 1). + for stale_index in [index for index in self._runtimes if index >= effective]: + self._teardown_world(stale_index) + for world_index in range(effective): + self._ensure_world(world_index) + # t3 / §4.1, p6-review-r1: "reconciles to exactly `instances` READY worlds" — + # `_ensure_world` only ever leaves a (re)built world `PREPARING` (§3's transition table + # promotes it later); promoted here via the same declared-readiness probe `healthy()` + # uses, so `provision()` itself returns worlds already at their reachable terminal state + # instead of leaving every caller to independently discover it must call `healthy()` + # first. A world left `READY`/`UNHEALTHY` by an earlier call is never re-probed here + # (mirrors `healthy()`'s own F6 "never promote except from PREPARING" rule). + for world_index in range(effective): + runtime = self._runtimes[world_index] + if runtime.state is RuntimeState.PREPARING: + # N2, p6-review-r2 (BLOCKER): polls (`_poll_runtime_health`), never samples once — + # nothing waits for the DAG's terminal process (`spawn_world` only waits on + # `depends_on` edges), so a single-shot probe here marked a bundle whose readiness- + # bearing process is terminal `UNHEALTHY` on every provision, regardless of how + # quickly it actually came up. + healthy_now = _poll_runtime_health(self._manifest, runtime, prober=context.prober) + runtime.state = RuntimeState.READY if healthy_now else RuntimeState.UNHEALTHY + + return [self._runtimes[index] for index in range(effective)] + + def _ensure_world(self, world_index: int) -> None: + """§4 rule 1: "completes or replaces partial/unhealthy worlds and never duplicates." A + world already `ready`/`preparing` is left alone (never duplicated); anything else (absent, + `unhealthy`, `stopped`) is (re)built from the sealed baseline via `_clone_or_reset_world` + — the same primitive `reset_world` uses, so a sick-world replace and a first-time clone are + one code path, not two. + """ + if self._manifest is None or self._context is None or self._build_output is None: + # m6, p6-review-r1: real precondition (only ever called from within `_provision_sync`, + # after all three are set) — typed, not a bare `assert` an `-O` run would strip. + raise ProcessRuntimeError( + "provision", _INTERNAL_INVARIANT_VIOLATED, + "_ensure_world called before manifest/context/build_output were established", + ) + existing = self._runtimes.get(world_index) + if existing is not None and existing.state in (RuntimeState.READY, RuntimeState.PREPARING): + return + + # N11, p6-review-r2. `current_world_index`: Q7, p6-review-r3 — a respawn reseals every + # tracked world's database, so every world OTHER than this one must be demoted for the + # scheduler to notice and repair it. + self._ensure_job_shared_handles_alive(current_world_index=world_index) + try: + result = _clone_or_reset_world( + self._manifest, world_index, context=self._context, baseline=self._build_output, + job_shared_handles=self._job_shared_handles, + existing_handles=self._world_handles.get(world_index, {}), + ) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): §4.6 — filesystem failures during provisioning are + # infrastructure. mkdir/chown/chmod for a (re)spawned process, uncaught this deep, + # used to raise bare out of `provision()`. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial # N4: never orphan a live engine. + raise ProcessRuntimeError("spawn", "spawn_failed", str(exc)) from exc + except BaseException as exc: + # N4, p6-review-r2 (MAJOR): a raise partway through `_clone_or_reset_world`/`spawn_ + # world` used to drop every ALREADY-(re)sealed/spawned handle of THIS world on the + # floor — nothing held it, so `close()`'s own `rmtree` of this world's data directory + # next ran against a still-live server. `exc.partial_handles` (set by `freeze_ + # baseline`/`_clone_or_reset_world`/`spawn_world`'s own try/finally) is whatever this + # (re)build managed before failing; published here so the next reconcile/close can + # terminate it instead. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial + raise + + self._world_handles[world_index] = result.handles + # N1, p6-review-r2 (BLOCKER): MUTATES the existing `EnvironmentRuntime` in place rather + # than constructing a replacement — v1.12 §4.5b's live-object model ("providers hand out + # live `EnvironmentRuntime` objects") reads as ONE object per world for the provider's + # whole life. Minting a new object every rebuild meant `reset()`'s own state write (m5) + # landed on an object no caller who captured an EARLIER reference would ever see again. + if existing is not None: + existing.runtime_id = new_runtime_id(self._bundle_digest, world_index) + existing.endpoints = result.endpoints + existing.state = RuntimeState.PREPARING + self._runtimes[world_index] = existing + else: + self._runtimes[world_index] = EnvironmentRuntime( + runtime_id=new_runtime_id(self._bundle_digest, world_index), + world_index=world_index, bundle_digest=self._bundle_digest, + state=RuntimeState.PREPARING, endpoints=result.endpoints, + ) + + def _drop_world_shared_databases(self, world_index: int) -> None: + """m4, p6-review-r1: reconciling W down (e.g. after a conformance-gate degrade from 3 to + 1) used to leave `w1`/`w2` behind forever on a job-shared `template_database` engine — + nothing ever DROPs a world's logical DB on teardown, only on REUSE (`_seal_world_store`'s + own `IF EXISTS`). A space leak, not a correctness one; best-effort (logged, not raised) so + a teardown-time drop failure never blocks the reconcile it is cleaning up after. + """ + if self._manifest is None or self._context is None: + return + store_by_process = _store_by_process_name(self._manifest) + for process in self._manifest.processes: + if not isinstance(process, ManagedProcess) or process.engine is not ManagedEngine.POSTGRES: + continue + store = store_by_process.get(process.name) + if store is None or store.baseline.strategy is not BaselineStrategy.TEMPLATE_DATABASE: + continue + if process.name not in self._job_shared_handles: + continue + credentials = self._context.credentials.get(process.name) + if credentials is None: + continue + port = self._context.port_plan.port_for(process.name, world_index) + world_db = f"w{world_index}" + try: + # N16, p6-review-r2 (MINOR): mirrors `_reset_template_database`'s own sibling + # call — a lingering backend on `world_db` (e.g. a scenario connection that never + # closed) otherwise blocks this DROP exactly the way it would block a reuse-time + # one, silently re-opening the space leak m4 closed. + _call_sql( + self._context.sql_runner, stage="teardown", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{world_db}' AND pid <> pg_backend_pid()" + ), + ) + _call_sql( + self._context.sql_runner, stage="teardown", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'DROP DATABASE IF EXISTS "{world_db}"', + ) + except ProcessRuntimeError as exc: + logger.warning("teardown: failed to drop %s on %s: %s", world_db, process.name, exc) + + def _teardown_world(self, world_index: int) -> None: + handles = self._world_handles.pop(world_index, {}) + # N12, p6-review-r2 (MINOR): reversed — `handles`' insertion order is `spawn_world`'s own + # `_topological_order` (dependencies first), so terminating it forward sends SIGTERM to a + # per-world engine while its own dependents (`tools-api`/`agent`) may still hold open + # connections, guaranteeing the full escalation wait every time. A dict's insertion order + # is preserved, so `reversed()` here IS reverse-topological order, no recomputation needed. + for name, handle in reversed(list(handles.items())): + if name not in self._job_shared_handles: + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait(handle.handle, prefer_interrupt=prefer_interrupt) + self._drop_world_shared_databases(world_index) # m4 + runtime = self._runtimes.pop(world_index, None) + if runtime is not None: + runtime.state = RuntimeState.STOPPED + if self._context is not None: + # m4, p6-review-r1: the per-world scratch tree (`/work/worlds/w/`) covers every + # process under this world, managed and source alike — left behind otherwise, the + # other space-leak half of the same finding. + world_dir = self._context.work_directory / "worlds" / f"w{world_index}" + if world_dir.exists(): + shutil.rmtree(world_dir, ignore_errors=True) + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + import asyncio + + await asyncio.to_thread(self._reset_sync, runtime) + + def _reset_sync(self, runtime: EnvironmentRuntime) -> None: + if self._manifest is None or self._context is None or self._build_output is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, "reset() called before provision()", + ) + world_index = runtime.world_index + self._ensure_job_shared_handles_alive(current_world_index=world_index) # N11/Q7. + try: + handles, sentinel_ok = reset_world( + self._manifest, world_index, context=self._context, baseline=self._build_output, + job_shared_handles=self._job_shared_handles, + existing_handles=self._world_handles.get(world_index, {}), + ) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): reset's own filesystem work is fundamentally "reseal this + # world's stores from baseline" — the `store_statement_failed` half of N9's mapping. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial # N4. + raise ProcessRuntimeError("reset", "store_statement_failed", str(exc)) from exc + except BaseException as exc: + partial = getattr(exc, "partial_handles", None) # N4, p6-review-r2. + if partial is not None: + self._world_handles[world_index] = partial + raise + self._world_handles[world_index] = handles + # m5, p6-review-r1: mutates THIS PROVIDER's own current record for `world_index`, never + # blindly overwrites it with whatever `EnvironmentRuntime` the caller happened to pass in + # — a caller holding a STALE object (e.g. one `_ensure_world` already replaced with a + # fresh `runtime_id`, before this `reset()` call was even made) used to be able to stomp + # the live record with stale endpoints/id. N1, p6-review-r2: after `_ensure_world` mutates + # in place instead of replacing, the provider's record and every caller's own reference + # for this world index are the SAME object for the provider's whole life — the fallback + # below (constructing from the caller's own `runtime`) is kept only for the unreachable + # case where this provider somehow has no record for the index at all. + current = self._runtimes.get(world_index) + if current is None: + current = runtime + self._runtimes[world_index] = current + if not sentinel_ok: + current.state = RuntimeState.UNHEALTHY + return + # M4, p6-review-r1: §4 point 3 — "`healthy` = declared `readiness` probes, not 'process is + # running.'" A passing sentinel alone used to be enough to mark `READY`; nothing waited for + # the LAST process in the world's `depends_on` DAG (nobody depends on the control service, + # so `spawn_world` never probes it) — the scheduler could dispatch a scenario against an + # agent still mid-boot. N2, p6-review-r2: polls (`_poll_runtime_health`), the same helper + # `provision()`'s own promotion uses, instead of sampling `probe_runtime_health` once. + healthy_now = _poll_runtime_health(self._manifest, current, prober=self._context.prober) + current.state = RuntimeState.READY if healthy_now else RuntimeState.UNHEALTHY + + async def close(self, *, work_directory: Path) -> None: + import asyncio + + await asyncio.to_thread(self._close_sync, work_directory) + + def _teardown_processes_and_directories(self, work_directory: Path) -> None: + """Shared by `close()` and M6's bundle-digest-change rebuild — terminates every live + handle and removes every job/world-scoped directory WITHOUT touching secrets or this + instance's own identity fields; `close()` clears those itself right after calling this, + while a digest-change rebuild is about to overwrite them with the new bundle's own values + a few lines later in `_provision_sync`. + + m10, p6-review-r1: every step is individually guarded — a failure removing ONE directory + (a permission error, a file still open) used to abort the whole method before the LATER + steps (clearing this instance's own dicts) ever ran, so the second `close()` §4.4 requires + to be a no-op was no longer one. Logged, never raised, so cleanup always reaches the end. + """ + # N12, p6-review-r2 (MINOR): reversed per world (dependents before the engine they depend + # on — see `_teardown_world`'s own note) and bounded by `self._close_wait_timeout_seconds` + # so `close()` can keep itself inside §0.7's 120s flush window regardless of how many + # per-world engines it has to wait out. + for handles in self._world_handles.values(): + for name, handle in reversed(list(handles.items())): + if name not in self._job_shared_handles: + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait( # M7: real wait, not just a sent signal. + handle.handle, timeout=self._close_wait_timeout_seconds, + prefer_interrupt=prefer_interrupt, + ) + for name, handle in reversed(list(self._job_shared_handles.items())): + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait( + handle.handle, timeout=self._close_wait_timeout_seconds, + prefer_interrupt=prefer_interrupt, + ) + for runtime in self._runtimes.values(): + runtime.state = RuntimeState.STOPPED + + for directory_name in ("build", "worlds", "managed"): + directory = work_directory / directory_name + try: + if directory.exists(): + shutil.rmtree(directory) + except OSError: + logger.warning("teardown: failed to remove %s; continuing", directory) + + self._job_shared_handles = {} + self._world_handles = {} + self._runtimes = {} + self._conformance_checked = False + + def _close_sync(self, work_directory: Path) -> None: + """§4 rule 4: idempotent hard-clean of everything — processes, data directories, build + trees, `secrets.json` if still present (the load in `_provision_sync` already deleted it + on the normal path; this is the "if still present" backstop §4.4 itself names, e.g. a + `close()` called before `provision()` ever ran). Idempotent by construction: every dict is + cleared at the end regardless of what failed along the way (m10). + """ + self._teardown_processes_and_directories(work_directory) + try: + self._secrets_path.unlink(missing_ok=True) # m9: no separate check-then-act .exists(). + except OSError: + logger.warning("close(): failed to unlink %s; continuing", self._secrets_path) + + self._manifest = None + self._bundle_digest = None + self._context = None + self._build_output = None + # Full reset, not just the spawn-state dicts `_teardown_processes_and_directories` + # already cleared — a provider reused for a genuinely NEW job after `close()` must load + # ITS OWN secrets file fresh, not silently run with the previous job's in-memory map (or + # none at all, since `_secrets_loaded` would otherwise still read `True`). + self._secret_values = {} + self._secret_purposes = {} + self._secrets_loaded = False + + def _ensure_job_shared_handles_alive(self, *, current_world_index: int | None = None) -> None: + """N11, p6-review-r2 (MAJOR): a job-shared `template_database` engine (§2b) backs EVERY + world at once, and nothing anywhere ever checked whether it was still running before + reusing it (`spawn_world`'s own `if name in handles: continue`) — an OOM-killed shared + postgres used to be carried forward as if healthy for the rest of the job, so every + subsequent `reset()`/reconcile raised `store_statement_failed` out of `_reset_template_ + database`'s own connection attempt against a dead process, `hosted_scheduler.py` swallowed + it, and no world ever recovered even though the engine's data directory was sitting intact + on disk the whole time. Called before `_clone_or_reset_world`/`reset_world` ever touch + `self._job_shared_handles`. `current_world_index` (Q7, p6-review-r3) is the world the + caller is already about to reconcile — passed through so a respawn can demote every OTHER + tracked world instead of leaving it `READY` against a database it just reset underneath. + """ + for name, handle in list(self._job_shared_handles.items()): + if not handle.handle.is_running(): + self._job_shared_handles[name] = self._respawn_dead_job_shared_engine( + name, current_world_index=current_world_index, + ) + + def _respawn_dead_job_shared_engine( + self, process_name: str, *, current_world_index: int | None = None, + ) -> SpawnedWorldProcess: + """Respawns from the SAME surviving data directory (never wiped — this is not a baseline + restore; the engine's own on-disk state is exactly what every world's logical database + still depends on), waits for readiness, then re-seals every world THIS PROVIDER currently + tracks from the template — the shared engine restarting is otherwise indistinguishable + from a fresh boot to postgres, but each world's own `w` database is a SEPARATE logical + database on that one instance and must be re-verified against the template again, same as + `_reset_template_database` already does for one world at a time. + """ + if self._manifest is None or self._context is None or self._build_output is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + "_respawn_dead_job_shared_engine called before manifest/context/build_output " + "were established", + ) + processes_by_name = {process.name: process for process in self._manifest.processes} + process = processes_by_name.get(process_name) + if not isinstance(process, ManagedProcess): + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process_name}: job-shared handle names a process that is not a managed engine", + process=process_name, + ) + context = self._context + build_output = self._build_output + port = context.port_plan.port_for(process_name, 0) + data_dir = managed_engine_data_dir(context.work_directory, process_name, world_index=None) + credentials = context.credentials.get(process_name) + try: + new_handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, + runner=context.runner, sync_run=context.sync_run, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + _wait_for_store_ready(self._manifest, process, port=port, context=context) + if process.engine is ManagedEngine.POSTGRES: + record = next( + (r for r in build_output.stores if r.process_name == process_name), None + ) + if record is not None: + live_credentials = _require_credentials( + credentials, stage="reset", process_name=process_name + ) + for world_index in self._runtimes: + _reset_template_database( + host="localhost", port=port, credentials=live_credentials, + world_db=f"w{world_index}", template_db=record.baseline_reference, + sql_runner=context.sql_runner, process_name=process_name, + ) + # Q7, p6-review-r3: the loop above just reset EVERY tracked world's database — + # demote every world other than the one the caller is already reconciling so + # the scheduler's next `reset()` repairs it, instead of handing out a `READY` + # world whose pooled connections point at a database dropped underneath it. + for idx, rt in self._runtimes.items(): + if idx != current_world_index: + rt.state = RuntimeState.UNHEALTHY + except (ProcessRuntimeError, OSError, shutil.Error) as exc: + raise ProcessRuntimeError( + "reset", "store_statement_failed", + f"{process_name}: job-shared engine died and could not be respawned: {exc}", + process=process_name, + ) from exc + return new_handle + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + """v1.12 §4.3's `RuntimeProvider.healthy` port, closing the gap p6-review-r2's task 2c + flagged: this class had `provision`/`reset`/`close` but no `healthy`, so it could not + structurally satisfy `runtime.py`'s `RuntimeProvider` Protocol. Demote-only, unlike the + module-level `healthy()` above — §3's transition table makes `unhealthy->ready` reachable + ONLY through a re-provision reconcile, and `provision()` already owns the `preparing-> + ready` promotion (`_poll_runtime_health`, before ever returning a world), so this port + method must never promote from ANY state, including `preparing`. `work_directory` is + accepted for §4 shape-compatibility and unused — the provider already holds its own + paths, same as `provision`'s own `contract` parameter. + + After N1 (p6-review-r2), `self._runtimes[world_index]` IS the same object every caller + holds for that world, so a single assignment on it is enough; `.get(..., runtime)` still + falls back to the caller's own object if this provider somehow has no record for the + index (unreachable in the normal `provision` -> `healthy` flow). + """ + import asyncio + + if self._manifest is None: + raise ProcessRuntimeError( + "healthy", _INTERNAL_INVARIANT_VIOLATED, "healthy() called before provision()", + ) + target = self._runtimes.get(runtime.world_index, runtime) + is_healthy = await asyncio.to_thread( + probe_runtime_health, self._manifest, target, prober=self._prober, + ) + if not is_healthy: + target.state = RuntimeState.UNHEALTHY + return is_healthy + + +__all__ = [ + "BuildOutput", + "CapabilityProber", + "EngineCredentials", + "EnvironmentRuntime", + "FreezeResult", + "PopenProcess", + "PortPlan", + "ProcessRunner", + "ProcessRuntimeError", + "ProcessRuntimeProvider", + "RabbitmqDefinitionsImporter", + "RabbitmqQueueDeclarer", + "RabbitmqQueueDeleter", + "RabbitmqQueueInspector", + "RedisCommandRunner", + "RuntimeEndpoint", + "RuntimeState", + "SpawnContext", + "SpawnedProcess", + "SpawnedWorldProcess", + "SqlRunner", + "StoreBaselineRecord", + "WorldSpawnResult", + "apply_seed_file", + "apply_store_seed", + "build_endpoints", + "build_process_tree", + "build_process_trees", + "build_tree_dir", + "check_sentinel", + "configuration_addresses_from_endpoints", + "default_capability_prober", + "default_process_runner", + "default_rabbitmq_definitions_importer", + "default_rabbitmq_queue_declare_and_publish", + "default_rabbitmq_queue_delete", + "default_rabbitmq_queue_inspector", + "default_redis_command_runner", + "default_sql_runner", + "default_user_resolver", + "freeze_baseline", + "generate_engine_credentials", + "healthy", + "managed_engine_data_dir", + "new_runtime_id", + "plan_ports", + "postgres_bootstrap_argv", + "postgres_daemon_argv", + "postgres_seed_argv", + "postgres_seed_env", + "probe_runtime_health", + "rabbitmq_conf_text", + "rabbitmq_daemon_argv", + "rabbitmq_daemon_env", + "rabbitmq_enabled_plugins_text", + "redis_daemon_argv", + "redis_seed_argv", + "render_capability_address", + "render_environment", + "render_template", + "reset_world", + "run_conformance_gate", "select_process_secrets", "spawn_managed_process", "spawn_source_process", "spawn_world", "wait_for_dependency", "world_scratch_dir", + "write_build_output", ] diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py index 2fa3b13b..6ab4fdc0 100644 --- a/tests/harness/test_process_preflight.py +++ b/tests/harness/test_process_preflight.py @@ -1,4 +1,4 @@ -"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.8. +"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.9. Every checklist item gets at least one rejection test carrying its named code, plus one clean accept-lane run of the full checklist. Bundles are built as real directories under `tmp_path` with @@ -495,9 +495,8 @@ def test_a_fixed_port_colliding_with_a_port_formula_band_is_rejected( """F11, p5-round1-review: `fixed_port` forces effective parallelism to 1, but the literal value was never checked against the provisioner's own port-formula bands — a bundle declaring `fixed_port: 14000` collides with a job-shared engine at ordinal 0, and the failure mode is an - opaque bind error inside a customer process, not a bundle rejection. `fixed_port_reserved` has - no §2e table entry yet (flagged for the owner in `PreflightError`'s own docstring) — the - S3 containment test below carries the same flag.""" + opaque bind error inside a customer process, not a bundle rejection. `fixed_port_reserved` is + in §2e's closed failure-code table as of v1.9.""" def mutate(body: dict[str, Any]) -> dict[str, Any]: body["processes"][1]["fixed_port"] = colliding_port @@ -878,16 +877,9 @@ def _raised_codes(source_text: str, callee_name: str, *, whole_first_argument: b return codes -# §2e's closed failure-code table (v1.8), transcribed verbatim — the single source of truth every +# §2e's closed failure-code table (v1.9), transcribed verbatim — the single source of truth every # raised code is checked against. Split exactly as the contract text splits it, purely for # reviewability against the spec; the test below treats it as one flat set. -# -# `fixed_port_reserved` (F11, p5-round1-review) is the one entry NOT actually in the frozen v1.8 -# table — flagged identically in `PreflightError`'s own docstring. The rule it guards (a -# `fixed_port` aliasing the provisioner's own port-formula bands) is real; the table predates it. -# Recorded here, transcribed alongside the real entries rather than hidden in a second set, so -# this test still does its job for every OTHER code — the one exception is a known, owner-facing -# gap, not silent drift. _SECTION_2E_CONTRACT_RULE_CODES = frozenset({ "compose_not_hosted", "engine_unsupported", "no_sql_store", "seed_missing", "seed_strategy_unsupported", "sentinel_shape_mismatch", "store_protocol_unsupported", @@ -898,7 +890,7 @@ def _raised_codes(source_text: str, callee_name: str, *, whole_first_argument: b "configuration_name_reserved", "sentinel_shape_invalid", "capability_unresolved", "service_unresolved", "control_service_unresolved", "process_name_duplicate", "inputs_digest_mismatch", - "fixed_port_reserved", # NOT in the frozen v1.8 table yet — see the note above. + "fixed_port_reserved", # §2e, v1.9. }) _SECTION_2E_MECHANICAL_CODES = frozenset({ "bundle_schema_unsupported", "bundle_manifest_invalid", "bundle_manifest_drifted", @@ -988,19 +980,35 @@ def _raised_codes_at_index(source_text: str, callee_name: str, *, index: int) -> return codes -# §2f's closed table (v1.8), transcribed verbatim. +# §2f's closed table (v1.8), transcribed verbatim, plus v1.10's two additions below. _SECTION_2F_CODES = frozenset({ "source_tree_unavailable", "build_failed", "runtime_unsupported", "spawn_failed", "depends_on_timeout", "unsupported_capability_protocol", + # `seed_failed` (v1.10, §2f): a §2c migration/seed step exited nonzero against the freshly + # started store — customer-authored content, deterministic, `environment` domain (never + # retried). Landed in the frozen table this version; no longer an out-of-vocabulary flag. + "seed_failed", + # `store_statement_failed` (v1.10, §2f): a managed store errored or rejected a provisioner- + # ISSUED statement (CREATE/DROP/ALTER DATABASE, sentinel or canary probe) after passing + # readiness — the harness's own statements, so a deterministic failure here is a harness/ + # engine fault, `infrastructure` domain (retryable), never `seed_failed` (that code is + # reserved for the customer's own migration/seed content). + "store_statement_failed", }) # `ProcessRuntimeError` also raises codes that are deliberately INTERNAL-only — each marks a # precondition `preflight_bundle` should already have made impossible (a placeholder token or a # missing credential preflight itself should have caught), so by the module's own docstring these # "never cross the outbound seam directly" and have no §2f entry to begin with. Excluded from # CONTAINMENT, not from extraction — a genuinely new internal code still surfaces in the raised -# set for a human to classify, since only these two documented names are exempted. +# set for a human to classify, since only these documented names are exempted. _INTERNAL_ONLY_RUNTIME_CODES = frozenset({ "internal_unknown_placeholder", "internal_missing_credentials", + # Phase 6: marks a bundle-shape/state invariant an earlier layer (the model layer, or this + # module's own baseline-freeze-before-clone ordering) should already guarantee — e.g. + # `reset()` called before `provision()`, or a store's backing service turning out not to be a + # `ManagedProcess` despite `bundle_v2`'s `store_service_not_managed` check. A bug to fix here, + # never a bundle defect the outbound seam needs a name for — same status as the other two. + "internal_invariant_violated", }) diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py index 203b73ed..60f752b1 100644 --- a/tests/harness/test_process_runtime.py +++ b/tests/harness/test_process_runtime.py @@ -1,5 +1,5 @@ """The execution half of the provisioner (`process_runtime.py`), per `hosted-execution-seams.md` -v1.8 §2b/§3/§4. Manifests here are built directly through `EnvironmentBundleV2.model_validate` — +v1.12 §2b/§3/§4/§5. Manifests here are built directly through `EnvironmentBundleV2.model_validate` — no digest sealing, no on-disk bundle — since every rule under test needs nothing but the parsed model's own field values plus the job-supplied inputs (instances, secrets, credentials). Digest and file-content verification are `test_process_preflight.py`'s job, not this module's. @@ -16,15 +16,26 @@ have and must not require — those paths are verified STRUCTURALLY: a fake `chown`/`user_resolver` is injected and the call it WOULD make is asserted, rather than performing the real privileged syscall. + +Phase 6 (seed/baseline/worlds/reset/conformance/provision/close) adds `SqlSpy` below: a fake +`SqlRunner` that records every `(dbname, statement)` call in order and simulates just enough +postgres semantics (CREATE/DROP/ALTER DATABASE, `to_regclass`, row counts) for the baseline-freeze +and reset state machines to run against — no real postgres anywhere, same rule as everything +above. `asyncio.run` drives every `async def` seam (`ProcessRuntimeProvider.provision`/`reset`/ +`close`) directly; there is no event loop fixture here, matching this repo's existing async test +style elsewhere in the harness suite. """ from __future__ import annotations +import asyncio import importlib.util +import json import subprocess import sys import time import types +from dataclasses import replace as dc_replace from pathlib import Path from typing import Any, Callable @@ -140,11 +151,17 @@ def _solo_port_plan(process_name: str, *, ordinal: int = 0) -> pr.PortPlan: class FakeHandle: - """A `SpawnedProcess` fake: no real subprocess, just a captured-output buffer.""" + """A `SpawnedProcess` fake: no real subprocess, just a captured-output buffer. `wait()` + reports the fake as exited the instant `terminate()`/`interrupt()`/`kill()` has been called — + no real process to actually wait on — which is enough for `_terminate_and_wait` (M7) to take + its happy path without ever escalating to `kill()` in a test. `StubbornHandle` (below, + N13-specific) is the deliberately-does-not-cooperate counterpart used to reach the kill branch.""" def __init__(self, output: str = "") -> None: self._output = output self.terminated = False + self.interrupted = False + self.killed = False def is_running(self) -> bool: return not self.terminated @@ -155,6 +172,20 @@ def captured_output(self) -> str: def terminate(self) -> None: self.terminated = True + def interrupt(self) -> None: + # N12, p6-review-r2: same instant-exit fake shape as `terminate()` — this fake exists to + # prove WHICH signal a caller preferred (`self.interrupted`), not to model postgres's own + # shutdown semantics. + self.interrupted = True + self.terminated = True + + def wait(self, timeout: float) -> bool: + return self.terminated + + def kill(self) -> None: + self.killed = True + self.terminated = True + class _FakePasswd: """Stands in for `pwd.struct_passwd` — only `pw_uid`/`pw_gid` are ever read by this module.""" @@ -1020,6 +1051,39 @@ def fake_sync_run(argv, **kwargs): assert any(uid == 2222 and gid == 3333 and path.endswith(".pwfile") for path, uid, gid in chowned) +def test_spawn_managed_process_chowns_a_pre_populated_data_dir_recursively(tmp_path: Path) -> None: + """B6, p6-review-r1: `data_dir` is empty on the very FIRST spawn (`initdb` itself creates + everything under it as the already-correct user), but `freeze_baseline`'s `datadir_copy` + snapshot and `_seal_world_store`'s restore both populate it via a COPY that runs as the + provisioner — every file underneath is provisioner-owned, and postgres refuses to start + unless the data directory AND ITS CONTENTS are owned by the effective user. Simulates that by + pre-populating nested content before the spawn call, same as a restored `datadir_copy` world + would look like at this point, and asserts every path underneath was chowned, not only the + top-level directory a single non-recursive `chown` call would have caught.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + (data_dir / "base" / "1").mkdir(parents=True) + (data_dir / "PG_VERSION").write_text("16\n") # already looks bootstrapped; initdb is skipped. + (data_dir / "base" / "1" / "1234").write_text("data") + chowned: list[tuple[str, int, int]] = [] + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + chowned_paths = {path for path, _, _ in chowned} + assert str(data_dir) in chowned_paths + assert str(data_dir / "base") in chowned_paths + assert str(data_dir / "base" / "1") in chowned_paths + assert str(data_dir / "base" / "1" / "1234") in chowned_paths + assert all(uid == 2222 and gid == 3333 for _, uid, gid in chowned) + assert (data_dir.stat().st_mode & 0o777) == 0o700 + + def test_default_process_runner_forwards_user_and_group_to_popen( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: @@ -1819,3 +1883,2389 @@ def test_probe_http_reports_not_ready_when_nothing_listens() -> None: assert pr.default_capability_prober( protocol=CapabilityProtocol.HTTP, host="localhost", port=1, path="/health" ) is False + + +# ================================================================================================= +# Phase 6: seed application, baseline freeze, world clone, reset, conformance gate, provision/close +# ================================================================================================= +# +# `_manifest()` above already carries one postgres store (`template_database`, job-shared, +# sentinel `SELECT 1` = `"1"`) — reused everywhere below unless a test needs a different strategy, +# in which case a `mutate` callback swaps `seed.stores[0].baseline` (and, where the row content +# under test matters, `migrations`/`seed_files`). `fake_sync_run`/`SqlSpy` below are this section's +# own `ProcessRunner`/`CapabilityProber`-equivalent fakes — no real postgres/redis/rabbitmq/docker +# anywhere, per the module docstring. + + +def _fake_sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + +def _recording_sync_run(calls: list[Any]) -> Callable[..., subprocess.CompletedProcess]: + def run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + return run + + +def _pg_spy_row(value: Any) -> list[tuple[Any, ...]]: + return [(value,)] + + +def _recording_prober(*, fail_times: int = 0) -> Callable[..., bool]: + """t1/N2, p6-review-r2: the promote-path fixtures used to inject `prober=lambda **kwargs: + True` — a fast pass that made `provision()`'s PREPARING->READY promotion and `reset()`'s + post-sentinel probe structurally unable to observe whether either one actually POLLED (N2) or + merely sampled once, the exact class of gap t1 was raised about for `_wait_for_store_ready`. + `fail_times=0` (the default) keeps every EXISTING test's prior instant-pass behavior + unchanged; a test proving the poll passes `fail_times=N` and asserts `len(prober.calls) > 1` + or that state ends up correct only after N+1 calls.""" + calls: list[dict[str, Any]] = [] + remaining = [fail_times] + + def prober(**kwargs: Any) -> bool: + calls.append(kwargs) + if remaining[0] > 0: + remaining[0] -= 1 + return False + return True + + prober.calls = calls # type: ignore[attr-defined] + return prober + + +class SqlSpy: + """A fake `SqlRunner`: records every `(dbname, statement)` call, in order — the "SQL spy" the + task calls for — and simulates just enough real postgres semantics for the baseline-freeze / + world-clone / reset / conformance-gate state machines to run against without a real server: + `CREATE`/`DROP`/`ALTER ... TEMPLATE` database bookkeeping, `to_regclass` existence checks, + `pg_tables`/`COUNT(*)` row counts, and a fixed answer for any `sentinel.query` a test seeds via + `answers`. `canary_leaks` deliberately makes the conformance canary visible across databases, + for the gate's FAIL-path test. + """ + + def __init__( + self, *, answers: dict[str, list[tuple[Any, ...]]] | None = None, canary_leaks: bool = False, + seeded_tables: dict[str, set[str]] | None = None, + ) -> None: + self.calls: list[tuple[str, str]] = [] + self.databases: dict[str, set[str]] = {} + self.row_data: dict[tuple[str, str], int] = {} + self.answers = answers or {} + self.canary_leaks = canary_leaks + # What a freshly `CREATE DATABASE`'d name starts with — simulates "the seed already + # landed" (a real `psql -f ...` would have populated it; the fake `sync_run` this spy + # runs alongside is a structural no-op, per the module docstring). + self.seeded_tables = seeded_tables or {} + + def __call__( + self, *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, + ) -> list[tuple[Any, ...]]: + self.calls.append((dbname, statement)) + body = statement.strip() + if read_only: + # N8, p6-review-r2: simulates postgres's own `SET default_transaction_read_only = on` + # rejecting a non-SELECT statement on a read-only session — the exact class of driver + # error `_call_sql` converts to `store_statement_failed`. Checked PER STATEMENT, not + # just the string's own prefix — psycopg3's simple-query protocol runs every `;`- + # separated statement in one unparameterized `execute()` call, which is exactly how a + # sentinel like `"SELECT 1; DROP TABLE riders"` smuggles a write past a naive + # starts-with-SELECT check while still LOOKING like a read at a glance. + for sub_statement in body.split(";"): + sub = sub_statement.strip() + if sub and not sub.upper().startswith(("SELECT", "WITH")): + raise RuntimeError(f"cannot execute {sub!r} in a read-only transaction") + if body in self.answers: + return self.answers[body] + if body == "SELECT 1": + return [(1,)] # `_manifest()`'s own default sentinel query. + if body.startswith("CREATE DATABASE") and "TEMPLATE" not in body: + name = body.split('"')[1] + self.databases[name] = set(self.seeded_tables.get(name, set())) + return [] + if "TEMPLATE" in body and body.startswith("CREATE DATABASE"): + source, target = body.split('"')[1], body.split('"')[3] + self.databases[source] = set(self.databases.get(target, set())) + return [] + if body.startswith("ALTER DATABASE") and "RENAME TO" in body: + old, new = body.split('"')[1], body.split('"')[3] + self.databases[new] = self.databases.pop(old, set()) + return [] + if body.startswith("DROP DATABASE"): + name = body.split('"')[-2] + self.databases.pop(name, None) + return [] + if body.startswith("CREATE TABLE") and "_alk_conformance" in body: + self.databases.setdefault(dbname, set()).add("_alk_conformance") + if self.canary_leaks: + leaked = "w1" if dbname == "w0" else "w0" + self.databases.setdefault(leaked, set()).add("_alk_conformance") + return [] + if body.startswith("SELECT to_regclass"): + present = "_alk_conformance" in self.databases.get(dbname, set()) + return [(present,)] + if body.startswith("SELECT tablename FROM pg_tables"): + return [(table,) for table in sorted(self.databases.get(dbname, set()))] + if body.startswith("SELECT COUNT(*)"): + table = body.split('"')[1] + return [(self.row_data.get((dbname, table), 0),)] + return [] + + +def _postgres_creds(m: EnvironmentBundleV2) -> dict[str, pr.EngineCredentials]: + return pr.generate_engine_credentials(m, token=lambda: "PW") + + +def _spawn_context( + manifest: EnvironmentBundleV2, *, instances: int = 2, bundle_dir: Path, sql_runner: Any = None, + redis_runner: Any = None, rabbitmq_inspector: Any = None, rabbitmq_declare: Any = None, + rabbitmq_delete: Any = None, rabbitmq_import: Any = None, runner: Any = None, + sync_run: Any = None, prober: Any = None, work_directory: Path, +) -> pr.SpawnContext: + port_plan = pr.plan_ports(manifest, instances=instances) + credentials = _postgres_creds(manifest) + return pr.SpawnContext( + work_directory=work_directory, port_plan=port_plan, credentials=credentials, + secret_values={}, secret_purposes={}, runner=runner or (lambda *a, **k: FakeHandle()), + sync_run=sync_run or _fake_sync_run, sql_runner=sql_runner or SqlSpy(), + redis_runner=redis_runner or (lambda **kwargs: None), + rabbitmq_inspector=rabbitmq_inspector or (lambda **kwargs: 0), + # m8, p6-review-r1: `SpawnContext`'s real defaults for these two now make an actual HTTP + # call — every test lane here must fake them explicitly, same rule as every other engine + # seam in this fixture, so a test that never overrides them never risks a real network + # call regardless of which store the conformance canary ends up preferring. + rabbitmq_declare=rabbitmq_declare or (lambda **kwargs: None), + rabbitmq_delete=rabbitmq_delete or (lambda **kwargs: None), + # N17, p6-review-r2: same rule — seeding now goes over HTTP too. + rabbitmq_import=rabbitmq_import or (lambda **kwargs: None), + bundle_dir=bundle_dir, + # Declared `readiness` probes are P5's own concern (`wait_for_dependency`/`healthy` + # already cover them elsewhere in this file) — a fast-pass fake here so Phase 6's own + # tests, which reuse `_manifest()`'s `tools` readiness entry incidentally, never block on + # a real HTTP probe against a `FakeHandle` that never actually opened a port. Also what + # B4's own `_wait_for_store_ready` polls — fast-pass here means every EXISTING freeze/ + # world-clone test keeps its prior (instant, no-wait) behavior unless it opts into a + # slower fake explicitly (t1's own observability test does exactly that; N2's new + # observability tests do the same one layer up, at the promote sites). + prober=prober or _recording_prober(), + ) + + +# --- §2c seed application ------------------------------------------------------------------------ + + +def test_postgres_seed_argv_shape() -> None: + argv = pr.postgres_seed_argv(port=14000, dbname="alk_baseline_postgres", user="harness", file=Path("db/schema.sql")) + assert argv == [ + "psql", "-h", "localhost", "-p", "14000", "-U", "harness", "-d", "alk_baseline_postgres", + "-v", "ON_ERROR_STOP=1", "-f", "db/schema.sql", + ] + + +def test_redis_seed_argv_has_no_file_flag_stdin_carries_the_commands() -> None: + assert pr.redis_seed_argv(port=15003) == ["redis-cli", "-h", "localhost", "-p", "15003"] + + +def test_default_rabbitmq_definitions_importer_posts_the_file_to_the_definitions_endpoint( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + """N17, p6-review-r2: `rabbitmqadmin import` is dropped entirely — seeding now POSTs the raw + definitions file straight to the management API's own `/api/definitions` endpoint, the same + HTTP-only seam every other rabbitmq call in this module already uses. No `rabbitmqadmin` + argv/binary anywhere in this path.""" + definitions_file = tmp_path / "mq" / "defs.json" + definitions_file.parent.mkdir(parents=True) + definitions_file.write_bytes(b'{"queues": []}') + creds = pr.EngineCredentials(username="harness", password="pw") + requests: list[Any] = [] + + def fake_urlopen(request: Any, timeout: float | None = None) -> Any: + requests.append(request) + + class _Response: + def __enter__(self) -> "_Response": + return self + + def __exit__(self, *exc: Any) -> None: + return None + + return _Response() + + monkeypatch.setattr(pr.urllib.request, "urlopen", fake_urlopen) + pr.default_rabbitmq_definitions_importer( + host="localhost", port=14002, credentials=creds, file=definitions_file, + ) + assert len(requests) == 1 + assert requests[0].full_url == "http://localhost:24002/api/definitions" + assert requests[0].data == b'{"queues": []}' + assert requests[0].get_header("Authorization") is not None + + +def test_apply_store_seed_runs_migrations_then_seed_files_in_listed_order(tmp_path: Path) -> None: + calls: list[Any] = [] + store = pr.StoreEntry.model_validate({ + "capability": "database", "migrations": ["db/001.sql", "db/002.sql"], + "seed_files": ["db/seed_a.sql", "db/seed_b.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1"}, + }) + creds = pr.EngineCredentials(username="harness", password="pw") + pr.apply_store_seed( + store, engine=pr.ManagedEngine.POSTGRES, bundle_dir=tmp_path, port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=_recording_sync_run(calls), + ) + files_in_order = [argv[argv.index("-f") + 1] for argv, _ in calls] + assert files_in_order == [ + str(tmp_path / "db/001.sql"), str(tmp_path / "db/002.sql"), + str(tmp_path / "db/seed_a.sql"), str(tmp_path / "db/seed_b.sql"), + ] + + +def test_apply_seed_file_postgres_env_keeps_path_and_adds_pgpassword(tmp_path: Path) -> None: + """A real bug caught by manual exercise before this test existed: `env=` REPLACES a child's + environment, not extends it — passing bare `{"PGPASSWORD": ...}` would drop `PATH` and make + `psql` unfindable outside `subprocess`'s narrow POSIX fallback.""" + calls: list[Any] = [] + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + pr.apply_seed_file( + pr.ManagedEngine.POSTGRES, tmp_path / "db/schema.sql", port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=_recording_sync_run(calls), + ) + _, kwargs = calls[0] + assert kwargs["env"]["PGPASSWORD"] == "s3cr3t" + assert "PATH" in kwargs["env"] + + +def test_apply_seed_file_redis_pipes_file_content_over_stdin(tmp_path: Path) -> None: + seed_file = tmp_path / "cache" / "seed.txt" + seed_file.parent.mkdir(parents=True) + seed_file.write_text("SET greeting hi\n") + calls: list[Any] = [] + pr.apply_seed_file( + pr.ManagedEngine.REDIS, seed_file, port=15003, dbname="", credentials=None, + process_name="cache", sync_run=_recording_sync_run(calls), + ) + argv, kwargs = calls[0] + assert argv == ["redis-cli", "-h", "localhost", "-p", "15003"] + assert kwargs["input"] == "SET greeting hi\n" + + +def test_apply_seed_file_raises_seed_failed_on_nonzero_exit(tmp_path: Path) -> None: + def failing_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(argv, 1, stdout="", stderr="syntax error") + + creds = pr.EngineCredentials(username="harness", password="pw") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.apply_seed_file( + pr.ManagedEngine.POSTGRES, tmp_path / "db/broken.sql", port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=failing_run, + ) + assert excinfo.value.code == "seed_failed" + assert excinfo.value.stage == "seed" + assert "syntax error" in str(excinfo.value) + + +# --- §2c sentinel checking ------------------------------------------------------------------------ + + +def test_check_sentinel_postgres_pass_and_fail() -> None: + store = _manifest().seed.stores[0] # {query: "SELECT 1", expected: "1"} + creds = pr.EngineCredentials(username="harness", password="pw") + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=lambda **kwargs: _pg_spy_row(1), + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=lambda **kwargs: _pg_spy_row(0), + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +def test_check_sentinel_passes_read_only_true_to_the_sql_runner() -> None: + """N8, p6-review-r2 (MAJOR): the sentinel is customer-authored content from an untrusted repo, + executed as `harness` — the role `initdb -U harness` makes the postgres SUPERUSER — over a + plain autocommit session. `check_sentinel` must mark this call `read_only=True` so `default_ + sql_runner` puts the session into a read-only transaction before running it.""" + store = _manifest().seed.stores[0] + creds = pr.EngineCredentials(username="harness", password="pw") + seen: dict[str, Any] = {} + + def recording_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + seen.update(kwargs) + return _pg_spy_row(1) + + pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=recording_sql_runner, + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert seen["read_only"] is True + + +def test_a_sentinel_query_attempting_a_write_fails(tmp_path: Path) -> None: + """N8, p6-review-r2 (MAJOR, task-required verification): the door B2 closed for `psql -f` + (dropping privilege for customer-authored SQL) was still open for `sentinel.query` — arbitrary + multi-statement SQL executed as the postgres superuser on a read-write session. `SqlSpy` + (`read_only=True`) rejects a non-`SELECT` statement exactly as a real `SET default_ + transaction_read_only = on` session would; a sentinel that tries to write must surface as a + typed `store_statement_failed`, not silently succeed. + """ + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "sentinel": { + "query": "SELECT 1; DROP TABLE riders", "expected": "1", + }, + }]}} + ) + store = manifest.seed.stores[0] + creds = pr.EngineCredentials(username="harness", password="pw") + sql_spy = SqlSpy() + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=sql_spy, + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_measure_postgres_row_counts_passes_read_only_true(tmp_path: Path) -> None: + """N8, p6-review-r2: the baseline row-count reads (`pg_tables`, `COUNT(*)`) are the other + read call site named in the fix — pinned directly rather than only indirectly through a + freeze-baseline integration test.""" + seen: list[bool] = [] + + def recording_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + seen.append(kwargs["read_only"]) + if kwargs["statement"].startswith("SELECT tablename"): + return [("riders",)] + return [(3,)] + + creds = pr.EngineCredentials(username="harness", password="pw") + counts = pr._measure_postgres_row_counts( + host="localhost", port=14000, credentials=creds, dbname="w0", + sql_runner=recording_sql_runner, process_name="postgres", + ) + assert counts == {"riders": 3} + assert seen == [True, True] # both the table listing AND the COUNT(*) itself. + + +def test_check_sentinel_redis_pass_and_fail() -> None: + store = pr.StoreEntry.model_validate({ + "capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}, + }) + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.REDIS, host="localhost", port=15003, dbname=None, + credentials=None, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: b"hi", + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.REDIS, host="localhost", port=15003, dbname=None, + credentials=None, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +def test_check_sentinel_rabbitmq_pass_and_fail() -> None: + store = pr.StoreEntry.model_validate({ + "capability": "queue", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "c" * 64}, + "sentinel": {"queue": "jobs", "expected_depth": 5}, + }) + creds = pr.EngineCredentials(username="harness", password="pw") + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.RABBITMQ, host="localhost", port=15003, dbname=None, + credentials=creds, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 5, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.RABBITMQ, host="localhost", port=15003, dbname=None, + credentials=creds, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +# --- §5.3 baseline freeze ------------------------------------------------------------------------- + + +def test_freeze_baseline_requires_bundle_dir() -> None: + manifest = _manifest() + ctx = pr.SpawnContext( + work_directory=Path("/x"), port_plan=pr.plan_ports(manifest, instances=1), + credentials={}, secret_values={}, secret_purposes={}, bundle_dir=None, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert excinfo.value.code == "internal_invariant_violated" + + +def test_freeze_baseline_waits_for_readiness_before_the_first_statement(tmp_path: Path) -> None: + """t1, p6-review-r1: B4's fix is structurally UNOBSERVABLE with a prober that always answers + `True` on the first call (`_spawn_context`'s own fast-pass default) — this pins that the wait + is REAL. A prober that fails twice before succeeding must be polled through all of them; the + manifest declares a `database` readiness entry with a tiny interval so the real `time.sleep` + between polls stays in the low milliseconds rather than the 30s/0.25s fallback default. + """ + manifest = _manifest( + lambda body: {**body, "readiness": [ + *body["readiness"], + {"capability": "database", "timeout_seconds": 5, "interval_seconds": 0.01}, + ]} + ) + probe_calls: list[dict[str, Any]] = [] + remaining_failures = [2] + + def flaky_prober(**kwargs: Any) -> bool: + probe_calls.append(kwargs) + if remaining_failures[0] > 0: + remaining_failures[0] -= 1 + return False + return True + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=flaky_prober, work_directory=tmp_path, + ) + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + # 2 failures + the passing call — proves `_wait_for_store_ready` actually looped rather than + # accepting the first `False` as good enough, or skipping the probe entirely. + assert len(probe_calls) >= 3 + assert remaining_failures[0] == 0 + assert all(call["protocol"] is pr.CapabilityProtocol.POSTGRES for call in probe_calls) + + +def test_freeze_baseline_readiness_timeout_is_a_typed_depends_on_timeout(tmp_path: Path) -> None: + """B4, p6-review-r1: a store that never becomes ready must fail typed (`depends_on_timeout`, + §2f — `infrastructure`, retryable), not hang forever or let the next statement run anyway.""" + manifest = _manifest( + lambda body: {**body, "readiness": [ + *body["readiness"], + {"capability": "database", "timeout_seconds": 0.05, "interval_seconds": 0.01}, + ]} + ) + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=lambda **kwargs: False, work_directory=tmp_path, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert excinfo.value.code == "depends_on_timeout" + + +def test_freeze_baseline_template_database_seeds_seals_and_measures_row_counts( + tmp_path: Path, +) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + }]}} + ) + # `seeded_tables` tells the spy what `CREATE DATABASE "alk_baseline_postgres"` should start + # with — simulating "the seed already landed," since the fake `sync_run` for `psql -f ...` + # below is a structural no-op (same rule as every other engine-binary call in this file). + sql_spy = SqlSpy(seeded_tables={"alk_baseline_postgres": {"riders"}}) + sql_spy.row_data[("alk_baseline_postgres", "riders")] = 3 + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=sql_spy, sync_run=_fake_sync_run, + work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert "postgres" in result.job_shared_handles # stays running — job-shared for the job. + record = result.build_output.stores[0] + assert record.strategy is pr.BaselineStrategy.TEMPLATE_DATABASE + assert record.baseline_reference == "alk_baseline_postgres" + assert record.row_counts == {"riders": 3} + + statements = [statement for _, statement in sql_spy.calls] + assert any(s.startswith("CREATE DATABASE") and "TEMPLATE" not in s for s in statements) + # migrations/seed_files apply via `sync_run`, not `sql_runner` — nothing to find in `statements` + # for them; asserting the CREATE precedes the ALTER is what is left to check here. + create_index = next(i for i, s in enumerate(statements) if s.startswith("CREATE DATABASE")) + alter_index = next(i for i, s in enumerate(statements) if "IS_TEMPLATE" in s) + assert create_index < alter_index + assert statements[alter_index] == ( + 'ALTER DATABASE "alk_baseline_postgres" WITH IS_TEMPLATE true ALLOW_CONNECTIONS false' + ) + + +def test_freeze_baseline_seed_commands_run_under_the_stores_declared_user(tmp_path: Path) -> None: + """t4 / B2, p6-review-r1: migration/seed files are customer-authored content applied through + `psql -f`, which honors backslash meta-commands (`\\!`, `\\copy ... program`) — must run under + the store's declared `svc-data` identity, never the provisioner's own uid, the same privilege + drop every other untrusted-content execution path in this module already gets + (`build_commands`, the managed-engine daemon itself).""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], "seed_files": [], + }]}} + ) + (tmp_path / "db").mkdir() + (tmp_path / "db" / "schema.sql").write_text("-- schema\n") + seed_calls: list[Any] = [] + + def recording_sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=recording_sync_run, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx = dc_replace( + ctx, user_resolver=_fake_user_resolver({"svc-data": (4444, 5555)}), + chown=lambda path, uid, gid: None, # a fake uid needs no real os.chown to prove the point. + ) + + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + psql_calls = [(argv, kwargs) for argv, kwargs in seed_calls if argv and argv[0] == "psql"] + assert psql_calls, "expected a psql -f seed invocation" + _, kwargs = psql_calls[0] + assert kwargs.get("user") == 4444 + assert kwargs.get("group") == 5555 + + +def test_freeze_baseline_datadir_copy_terminates_the_bootstrap_and_snapshots_the_datadir( + tmp_path: Path, +) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + handles: list[FakeHandle] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + handle = FakeHandle() + handles.append(handle) + return handle + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert result.job_shared_handles == {} # datadir_copy is never job-shared. + assert len(handles) == 1 + assert handles[0].terminated is True # the bootstrap instance is stopped after the copy. + record = result.build_output.stores[0] + assert record.strategy is pr.BaselineStrategy.DATADIR_COPY + assert Path(record.baseline_reference) == tmp_path / "managed" / "postgres.baseline" + + +def test_freeze_baseline_datadir_copy_redis_issues_save_before_terminating( + tmp_path: Path, +) -> None: + """Q1, p6-review-r3 (MAJOR): `redis_daemon_argv` disables save points (`--save ""`) — a bare + SIGTERM shutdown persists nothing, so `_freeze_one_store`'s `DATADIR_COPY` branch must issue + an explicit synchronous `SAVE` immediately before terminating, or the copied data dir every + world clones/resets from is an empty baseline.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + events: list[str] = [] + + class RecordingHandle(FakeHandle): + def terminate(self) -> None: + events.append("TERMINATE") + super().terminate() + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + return RecordingHandle() + + def redis_runner(*, host: str, port: int, command: Any) -> Any: + events.append(command[0]) + return "hi" if command[0] == "GET" else None + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, redis_runner=redis_runner, + work_directory=tmp_path, + ) + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + # only "cache" (redis, datadir_copy) is ever terminated at freeze time — postgres stays + # running (template_database, job-shared), so this is unambiguously the redis command order. + assert events[-2:] == ["SAVE", "TERMINATE"] + + +def test_freeze_baseline_empty_strategy_is_a_no_op_capture(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + spawned_names: list[str] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + return FakeHandle() + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + spawned_names.append(argv[0]) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, sync_run=sync_run, work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + cache_record = next(r for r in result.build_output.stores if r.process_name == "cache") + assert cache_record.strategy is pr.BaselineStrategy.EMPTY + assert cache_record.baseline_reference == "" + assert cache_record.row_counts == {} + assert "redis-cli" not in spawned_names # nothing seeded at freeze time for `empty`. + assert "cache" not in result.job_shared_handles + + +def test_write_build_output_shape(tmp_path: Path) -> None: + build_output = pr.BuildOutput( + bundle_digest="sha256:" + "0" * 64, + stores=[pr.StoreBaselineRecord( + capability="database", process_name="postgres", engine=pr.ManagedEngine.POSTGRES, + strategy=pr.BaselineStrategy.TEMPLATE_DATABASE, inputs_digest="sha256:" + "a" * 64, + baseline_reference="alk_baseline_postgres", row_counts={"riders": 3}, + )], + conformance=True, conformance_reason=None, + ) + target = pr.write_build_output(tmp_path, build_output) + assert target == tmp_path / "artifacts" / "build.json" + payload = json.loads(target.read_text()) + assert payload["bundle_digest"] == build_output.bundle_digest + assert payload["conformance"] is True + assert payload["stores"] == [{ + "capability": "database", "process_name": "postgres", "engine": "postgres", + "strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64, + "baseline_reference": "alk_baseline_postgres", "row_counts": {"riders": 3}, + }] + + +# --- §4.2 world clone + reset ---------------------------------------------------------------------- + + +def _frozen_template_database( + manifest: EnvironmentBundleV2, tmp_path: Path, +) -> tuple[pr.FreezeResult, pr.SpawnContext, SqlSpy]: + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + sql_spy.calls.clear() + return result, ctx, sql_spy + + +def test_world_clone_template_database_issues_terminate_drop_create_template( + tmp_path: Path, +) -> None: + manifest = _manifest() + freeze_result, ctx, sql_spy = _frozen_template_database(manifest, tmp_path) + + result = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert set(result.handles) == {"postgres", "tools-api", "agent"} + assert result.handles["postgres"] is freeze_result.job_shared_handles["postgres"] # reused. + + statements = [statement for _, statement in sql_spy.calls] + assert statements == [ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE datname = 'w0' AND pid <> pg_backend_pid()", + 'DROP DATABASE IF EXISTS "w0"', + 'CREATE DATABASE "w0" TEMPLATE "alk_baseline_postgres"', + ] + + +def test_world_clone_datadir_copy_copies_the_baseline_then_renames_to_wn(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + baseline_dir = Path(freeze_result.build_output.stores[0].baseline_reference) + assert baseline_dir.is_dir() + sql_spy.calls.clear() + + copy_calls: list[tuple[Path, Path]] = [] + real_copy = pr._copytree_preserving_symlinks + + def recording_copy(src: Path, dst: Path) -> None: + copy_calls.append((src, dst)) + real_copy(src, dst) + + ctx2 = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx2 = dc_replace(ctx2, copy=recording_copy) + + result = pr._clone_or_reset_world( + manifest, 0, context=ctx2, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert copy_calls == [(baseline_dir, tmp_path / "worlds" / "w0" / "postgres")] + statements = [statement for _, statement in sql_spy.calls] + assert statements == ['ALTER DATABASE "alk_baseline_postgres" RENAME TO "w0"'] + assert "postgres" in result.handles + + +def test_world_clone_empty_strategy_reseeds_on_every_clone(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + seed_calls: list[Any] = [] + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append(argv) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=sync_run, work_directory=tmp_path, + ) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert not any("redis-cli" in argv for argv in seed_calls) # nothing at freeze time. + + pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert any("redis-cli" in argv for argv in seed_calls) # (re)established at clone time. + + +def test_world_clone_empty_strategy_reseed_runs_under_the_stores_declared_user( + tmp_path: Path, +) -> None: + """t4 / B2, p6-review-r1: the `empty`-strategy reseed path (`_seal_world_store`'s fallback + branch) is a SEPARATE call site from `freeze_baseline`'s own seed application — both must + drop privilege, not just the one exercised by the freeze-time test above.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + seed_calls: list[Any] = [] + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=sync_run, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx = dc_replace( + ctx, user_resolver=_fake_user_resolver({"svc-data": (6666, 7777)}), + chown=lambda path, uid, gid: None, # a fake uid needs no real os.chown to prove the point. + ) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + redis_calls = [(argv, kwargs) for argv, kwargs in seed_calls if "redis-cli" in argv] + assert redis_calls + _, kwargs = redis_calls[-1] + assert kwargs.get("user") == 6666 + assert kwargs.get("group") == 7777 + + +def test_world_clone_empty_strategy_requires_bundle_dir_to_reseed(tmp_path: Path) -> None: + """Mirrors `freeze_baseline`'s own `bundle_dir` guard: an empty-strategy store re-seeded on + every clone must fail typed if the context somehow carries no bundle directory, never silently + resolve `seed_files` against the wrong (cwd) directory.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + ctx = _spawn_context(manifest, bundle_dir=tmp_path, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + from dataclasses import replace as dc_replace + ctx_no_bundle_dir = dc_replace(ctx, bundle_dir=None) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._clone_or_reset_world( + manifest, 0, context=ctx_no_bundle_dir, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert excinfo.value.code == "internal_invariant_violated" + + +def test_reset_world_terminates_only_per_world_handles(tmp_path: Path) -> None: + manifest = _manifest() + freeze_result, ctx, sql_spy = _frozen_template_database(manifest, tmp_path) + world0 = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + old_tools_api = world0.handles["tools-api"] + old_agent = world0.handles["agent"] + shared_postgres = world0.handles["postgres"] + + handles, healthy = pr.reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles=world0.handles, + ) + assert old_tools_api.handle.terminated is True + assert old_agent.handle.terminated is True + assert shared_postgres.handle.terminated is False # job-shared — stays up across a reset. + assert healthy is True # SqlSpy answers `SELECT 1` -> 1 by default (no `answers` override). + + +def test_reset_world_sentinel_failure_reports_unhealthy_without_raising(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy() # passes at freeze time (default "SELECT 1" -> 1) — m3's own freeze-time + # sentinel check must NOT be what fails this test; only `reset_world`'s is under test here. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world0 = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + # Corrupted AFTER the (passing) freeze-time check, simulating something breaking the world's + # own state before its reset — never matches the sentinel's "1" from here on. + sql_spy.answers["SELECT 1"] = [(0,)] + handles, healthy = pr.reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles=world0.handles, + ) + assert healthy is False # §4.2: a sentinel failure is reported, never raised. + + +# --- §4 conformance gate ------------------------------------------------------------------------- + + +def test_first_canary_store_prefers_postgres_over_redis_and_rabbitmq() -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + {"capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "k", "expected": "v"}}, + body["seed"]["stores"][0], + ]}} + ) + store = pr._first_canary_store(manifest) + assert store is not None + assert store.capability == "database" # postgres, even though redis is listed first. + + +def test_first_canary_store_is_none_with_no_seed_block() -> None: + manifest = _manifest(lambda body: {**body, "runtime": {**body["runtime"], "kind": "external"}, + "processes": [], "seed": None}) + assert pr._first_canary_store(manifest) is None + + +def _manifest_with_rabbitmq_store() -> EnvironmentBundleV2: + return _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "queue", "kind": "managed", "engine": "rabbitmq", "version": "3.13", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "queue": {"protocol": "amqp", "service": "queue", "configuration_name": "QUEUE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "queue", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "d" * 64}, + "sentinel": {"queue": "jobs", "expected_depth": 0}}, + ]}} + ) + + +def test_run_canary_probe_rabbitmq_declares_publishes_and_inspects_without_deleting( + tmp_path: Path, +) -> None: + """m8, p6-review-r1: previously only ever INSPECTED a queue nobody had declared — a vacuous + pass regardless of real isolation. Pins the real sequence: declare+publish in world 0, inspect + world 1 — real HTTP calls faked, never skipped. + + N15, p6-review-r2 (MINOR): no delete anymore. The delete used to run HERE, before `run_ + conformance_gate`'s own `reset_world` calls for both worlds — by the time `_verify_canary_ + absent` later re-checked world 0, the queue was already gone from THIS unrelated step, + proving nothing about whether the reset itself actually worked. Cleanup is now the reset's + job (world 0's rabbitmq is always `datadir_copy`, wiped and restarted from the pristine + baseline snapshot on every reset), the same mechanism postgres/redis's own canary already + relied on.""" + manifest = _manifest_with_rabbitmq_store() + store = manifest.seed.stores[1] # the rabbitmq store. + declare_calls: list[dict[str, Any]] = [] + delete_calls: list[dict[str, Any]] = [] + inspected_ports: list[int] = [] + + def rabbitmq_declare(**kwargs: Any) -> None: + declare_calls.append(kwargs) + + def rabbitmq_delete(**kwargs: Any) -> None: + delete_calls.append(kwargs) + + def rabbitmq_inspector(**kwargs: Any) -> int: + inspected_ports.append(kwargs["port"]) + return 0 # world 1 never saw the canary — real isolation. + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, work_directory=tmp_path, + rabbitmq_declare=rabbitmq_declare, rabbitmq_delete=rabbitmq_delete, + rabbitmq_inspector=rabbitmq_inspector, + ) + result = pr._run_canary_probe(manifest, store, pr.ManagedEngine.RABBITMQ, context=ctx) + assert result is True + assert len(declare_calls) == 1 + assert declare_calls[0]["queue"] == "_alk_conformance" + assert delete_calls == [] # N15: cleanup deferred to the subsequent reset, not done here. + port0 = ctx.port_plan.port_for("queue", 0) + port1 = ctx.port_plan.port_for("queue", 1) + assert declare_calls[0]["port"] == port0 # declared in world 0. + assert inspected_ports == [port1] # inspected in world 1. + + +def test_run_canary_probe_rabbitmq_fails_when_the_queue_leaks_to_world_1(tmp_path: Path) -> None: + """m8: a leaked canary (world 1's inspector reports a message that should not be there) must + fail the probe, proving the check is not vacuous in the other direction either.""" + manifest = _manifest_with_rabbitmq_store() + store = manifest.seed.stores[1] + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, work_directory=tmp_path, + rabbitmq_declare=lambda **kwargs: None, rabbitmq_delete=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 1, # world 1 sees a message: a real leak. + ) + result = pr._run_canary_probe(manifest, store, pr.ManagedEngine.RABBITMQ, context=ctx) + assert result is False + + +def test_default_rabbitmq_queue_inspector_treats_404_as_zero_depth( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """m8, p6-review-r1: a 404 for a nonexistent queue IS "empty" for the canary/sentinel's own + purposes — the management API's way of saying so, not an error the gate's 'never raises' + promise should have had to survive only by luck (unreachable via postgres-first preference).""" + import urllib.error + + def fake_urlopen(request: Any, timeout: float | None = None) -> Any: + raise urllib.error.HTTPError(request.full_url, 404, "Not Found", None, None) + + monkeypatch.setattr(pr.urllib.request, "urlopen", fake_urlopen) + creds = pr.EngineCredentials(username="harness", password="pw") + depth = pr.default_rabbitmq_queue_inspector( + host="localhost", port=15003, credentials=creds, queue="_alk_conformance", + ) + assert depth == 0 + + +def test_conformance_gate_passes_when_worlds_are_really_isolated(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy() # `canary_leaks=False` — the default; a real isolation bug would leak. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (True, None) + + +def test_conformance_gate_fails_and_never_raises_when_isolation_is_broken(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy(canary_leaks=True) # simulates a real cross-world isolation bug. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +def test_conformance_gate_is_vacuously_true_with_no_canary_store(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "runtime": {**body["runtime"], "kind": "external"}, + "processes": [], "seed": None}) + ctx = _spawn_context(manifest, bundle_dir=tmp_path, work_directory=tmp_path) + build_output = pr.BuildOutput(bundle_digest=manifest.digest, stores=[]) + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=build_output, job_shared_handles={}, world_handles={}, + ) + assert (passed, reason) == (True, None) + + +# --- §4 provision / reset / close: ProcessRuntimeProvider ------------------------------------------- + + +def _sql_spy_provider(**overrides: Any) -> pr.ProcessRuntimeProvider: + kwargs: dict[str, Any] = dict( + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, sql_runner=SqlSpy(), + prober=_recording_prober(), # N2, p6-review-r2: see `_spawn_context`'s own comment. + rabbitmq_declare=lambda **kwargs: None, rabbitmq_delete=lambda **kwargs: None, + rabbitmq_import=lambda **kwargs: None, + ) + kwargs.update(overrides) + return pr.ProcessRuntimeProvider(**kwargs) + + +def _provision_dirs(tmp_path: Path) -> tuple[Path, Path]: + """B1, p6-review-r1: `source` (the untrusted checkout) and `bundle_dir` (the verified bundle + root) are now GENUINELY DISTINCT directories, never the same path passed twice — a regression + back to conflating them would be invisible if every test fixture kept them identical (t2).""" + source = tmp_path / "source" + (source / "services" / "tools-api").mkdir(parents=True) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir(parents=True) + return source, bundle_dir + + +def test_provision_reconciles_to_exactly_w_ready_worlds(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0, 1, 2] + assert len({runtime.runtime_id for runtime in runtimes}) == 3 # never duplicated. + # t3, p6-review-r1: §4.1 says `provision` "reconciles to exactly `instances` READY worlds" — + # previously true only of the COUNT, never checked that a returned world is actually `ready` + # (it was `preparing`, forever, until some OTHER caller happened to call `healthy()`). + assert all(runtime.state is pr.RuntimeState.READY for runtime in runtimes) + ports = {runtime.endpoints["tools"].address for runtime in runtimes} + assert len(ports) == 3 # each world's own tools-api port, all distinct. + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["conformance"] is True + # m1, p6-review-r1: both degrade causes now land on build.json, not just the gate's own. + assert build_output["requested_parallelism"] == 3 + assert build_output["effective_parallelism"] == 3 + assert build_output["degrade_reason"] is None + + +def test_provision_reads_seed_files_from_bundle_dir_not_source(tmp_path: Path) -> None: + """B1, p6-review-r1: §2c seed/migration paths are bundle-relative and must resolve against + the VERIFIED bundle directory, never the untrusted checkout — a bundle declares `migrations: + ["db/schema.sql"]`; preflight hashes/scans `/db/schema.sql`, so reading the same + relative path from `source` instead would execute bytes nothing ever verified. `source` and + `bundle_dir` are genuinely different directories and the seed file exists ONLY under + `bundle_dir` (t2) — a regression back to `bundle_dir=source` is caught by asserting the + RECORDED `-f` argument, not merely that the fake `sync_run` returned success (which it always + does regardless of the path).""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], "seed_files": [], + }]}} + ) + source, bundle_dir = _provision_dirs(tmp_path) + (bundle_dir / "db").mkdir(parents=True) + (bundle_dir / "db" / "schema.sql").write_text("-- schema\n") + + calls: list[Any] = [] + provider = _sql_spy_provider( + sync_run=_recording_sync_run(calls), secrets_path=tmp_path / "secrets.json", + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + seed_calls = [argv for argv, _ in calls if argv and argv[0] == "psql"] + assert seed_calls, "expected a psql -f seed invocation" + applied_file = seed_calls[0][seed_calls[0].index("-f") + 1] + assert applied_file == str(bundle_dir / "db" / "schema.sql") + + +def test_provision_is_idempotent_for_the_same_job_identity(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + spawn_count: list[int] = [0] + + def counting_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawn_count[0] += 1 + return FakeHandle() + + provider = _sql_spy_provider(runner=counting_runner, secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + count_after_first = spawn_count[0] + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.runtime_id for r in first] == [r.runtime_id for r in second] + assert spawn_count[0] == count_after_first # nothing re-spawned; every world already `ready`. + + +def test_provision_hosted_mode_require_declared_user_true_fails_typed_when_unresolvable( + tmp_path: Path, +) -> None: + """P5 ledger carry-forward, exercised end to end through `provision()` rather than only at + `build_process_tree`/`spawn_source_process` directly: the hosted path passes + `require_declared_user=True`, and a snapshot that somehow lacks a declared `svc-*` user must + fail typed, never silently fall back to running unprivileged. M2, p6-review-r1: `True` is now + `provision()`'s own DEFAULT (fail-closed) — passed explicitly here anyway so the test still + reads as pinning the behavior on its own, not merely relying on the default not having moved.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + user_resolver=lambda name: None, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=True, + )) + assert excinfo.value.code == "spawn_failed" + + +def test_provision_require_declared_user_defaults_to_true(tmp_path: Path) -> None: + """M2, p6-review-r1: `ProcessRuntimeProvider` is hosted-only — a caller that omits the keyword + entirely must still fail closed, not silently fall back to running everything unprivileged as + the harness's own `svc-control`. Mirrors the test above but never passes the argument.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + user_resolver=lambda name: None, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + )) + assert excinfo.value.code == "spawn_failed" + + +def test_provision_conformance_degrade_persists_across_a_later_reconcile_call( + tmp_path: Path, +) -> None: + """A real bug caught by manual exercise before this test existed: `effective` is recomputed + fresh from `port_plan` on every call, so a gate failure decided on call 1 was silently + forgotten on call 2 — worlds beyond index 0 came back even though the gate never re-ran.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + sql_runner=SqlSpy(canary_leaks=True), secrets_path=tmp_path / "secrets.json", + ) + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in first] == [0] + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in second] == [0] + assert first[0].runtime_id == second[0].runtime_id + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["degrade_reason"] == "conformance_gate_failed" # m1 + + +def test_provision_tears_down_before_rebuilding_on_a_bundle_digest_change(tmp_path: Path) -> None: + """M6, p6-review-r1: a bundle-digest change (a re-sealed bundle mid-attempt) used to reassign + this instance's own identity straight over the PREVIOUS job's still-running processes and + still-allocated ports — §4.1's "never duplicates" broken in the one case this branch exists + for. Every previously-live handle (job-shared and per-world) must be terminated BEFORE the new + manifest's own build/freeze ever runs.""" + manifest_a = _manifest() + manifest_b = _manifest(lambda body: {**body, "digest": "sha256:" + "9" * 64}) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + + first = asyncio.run(provider.provision( + manifest_a, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + old_world_handles = [h for world in provider._world_handles.values() for h in world.values()] + old_shared_handles = list(provider._job_shared_handles.values()) + assert old_world_handles or old_shared_handles # something is actually running to tear down. + + second = asyncio.run(provider.provision( + manifest_b, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert all(h.handle.terminated for h in old_world_handles) + assert all(h.handle.terminated for h in old_shared_handles) + assert second[0].bundle_digest == manifest_b.digest + assert second[0].bundle_digest != first[0].bundle_digest + + +def test_provision_recovers_a_sick_world_via_re_call(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = first[0] + old_agent_handle = provider._world_handles[0]["agent"] + old_runtime_id = runtime.runtime_id # N1, p6-review-r2: snapshotted BEFORE mutation — after + # N1, `second[0]` and `runtime` are the SAME object, so comparing `second[0].runtime_id != + # runtime.runtime_id` post-rebuild would compare an attribute against itself. + runtime.state = pr.RuntimeState.UNHEALTHY + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert second[0] is runtime # N1: mutated in place, never replaced. + assert second[0].runtime_id != old_runtime_id # rebuilt, not left unhealthy. + # t3, p6-review-r1: `provision()` now promotes a freshly-(re)built world out of `PREPARING` + # before returning (the same declared-readiness probe `healthy()` uses) — was `PREPARING` + # before this fix pass, unconditionally. + assert second[0].state is pr.RuntimeState.READY + assert old_agent_handle.handle.terminated is True + + +def test_reset_transitions_ready_or_unhealthy_per_sentinel(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + asyncio.run(provider.reset(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.READY + + +def test_ensure_world_mutates_the_same_environment_runtime_object_across_rebuilds( + tmp_path: Path, +) -> None: + """N1, p6-review-r2 (BLOCKER), superseding m5's own test: v1.12 §4.5b's live-object model + ("providers hand out live `EnvironmentRuntime` objects") reads as ONE object per world for + the provider's whole life. `_ensure_world` used to mint a brand-new `EnvironmentRuntime` on + every rebuild (m5's own fixture exercised exactly that, holding a deliberately STALE, + replaced object) — under that shape, a caller holding an EARLIER reference (e.g. `hosted_ + scheduler.py`'s own pool entry, captured right after the first `provision()` call) could never + see a later rebuild's state land anywhere it could observe. Pins that a sick-world recovery + rebuild MUTATES the object a caller is already holding, in place — fresh `runtime_id`/ + `endpoints`/`state`, same Python object, visible through the ORIGINAL reference with no + re-fetch required. `reset()`'s own state write (m5) is a trivial consequence once there is + only one object to write to. + """ + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + held = first[0] + old_runtime_id = held.runtime_id + held.state = pr.RuntimeState.UNHEALTHY # simulate the scheduler demoting it (§4.5b). + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert second[0] is held # the SAME object — never replaced. + assert held.runtime_id != old_runtime_id # rebuilt: a fresh identity... + assert held.state is pr.RuntimeState.READY # ...and promoted — both visible through `held`. + + asyncio.run(provider.reset(held, work_directory=tmp_path)) + assert provider._runtimes[0] is held # reset() writes through the SAME object too. + assert held.state is pr.RuntimeState.READY + + +def test_close_is_idempotent_and_removes_secrets_and_data_directories(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "run-secrets.json" + secrets_path.write_text("{}") + provider = _sql_spy_provider(secrets_path=secrets_path) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + handles = [handle for world in provider._world_handles.values() for handle in world.values()] + assert handles # something is actually running before close(). + # B3, p6-review-r1: the secrets file is gone the moment `provision()` loaded it — long before + # `close()` ever runs (t5). `close()`'s own unlink is the "if still present" backstop only. + assert not secrets_path.exists() + + asyncio.run(provider.close(work_directory=tmp_path)) + assert all(handle.handle.terminated for handle in handles) + assert not secrets_path.exists() + assert not (tmp_path / "build").exists() + assert not (tmp_path / "worlds").exists() + assert not (tmp_path / "managed").exists() + for runtime in runtimes: + assert runtime.state is pr.RuntimeState.STOPPED + + asyncio.run(provider.close(work_directory=tmp_path)) # must not raise the second time. + + +# --- B3/§0.3, p6-review-r1: secrets lifetime and injection through provision() ----------------- + + +def test_secrets_are_loaded_and_the_file_is_gone_before_the_first_process_spawns( + tmp_path: Path, +) -> None: + """t5: pins §0.3's lifetime rule end to end — "the provisioner loads this file into memory at + startup and deletes it immediately after loading, BEFORE ANY CUSTOMER PROCESS STARTS." Records + whether the secrets file still existed at the moment of the FIRST spawn call (build step, + managed engine, or source process — whichever the provisioner reaches first).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + existed_at_first_spawn: list[bool] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + existed_at_first_spawn.append(secrets_path.exists()) + return FakeHandle() + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + existed_at_first_spawn.append(secrets_path.exists()) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + provider = _sql_spy_provider(runner=runner, sync_run=sync_run, secrets_path=secrets_path) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert existed_at_first_spawn, "expected at least one spawn/sync_run call" + assert not any(existed_at_first_spawn), "secrets.json must be gone before the FIRST spawn" + assert not secrets_path.exists() + + +def test_secrets_are_re_injected_from_memory_on_reset(tmp_path: Path) -> None: + """t5: §0.3 — "the in-memory map lives for the whole job — `reset` restarts... re-inject from + memory." The file is deleted after the first `provision()`, so a `reset()` that still lands + the secret in the agent's env can only be reading it from the in-memory map, not the disk.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + envs_by_argv0: dict[str, dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + # N10, p6-review-r2: no `/work/job.json` exists in this fixture's `tmp_path` — the + # constructor override stands in for it, the same way a local/test lane would. + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, + ) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert not secrets_path.exists() # gone after the first provision() — nothing left to re-read. + + envs_by_argv0.clear() + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + agent_envs = [env for argv, env in envs_by_argv0.items() if argv == ("python3", "agent.py")] + assert agent_envs, "expected the agent process to be respawned by reset()" + assert agent_envs[0].get("LIVEKIT_API_KEY") == "abc123" + + +def test_secret_injection_end_to_end_through_provision(tmp_path: Path) -> None: + """t6: no prior test covered secret injection through `provision()` at all — `select_process_ + secrets` was only ever tested in isolation. A real (on-disk) secrets.json, loaded by a real + `provision()` call, must land the claimed alias in the claiming process's env and must NOT + land in a process that never claimed `target_provider`.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + envs_by_argv0: dict[tuple[str, ...], dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, # N10, p6-review-r2. + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + agent_env = envs_by_argv0[("python3", "agent.py")] # claims `target_provider` in `_manifest()`. + assert agent_env.get("LIVEKIT_API_KEY") == "abc123" + tools_env = envs_by_argv0[("node", "server.js")] # claims no purpose in `_manifest()`. + assert "LIVEKIT_API_KEY" not in tools_env + + +# --- N10, p6-review-r2 (MAJOR): secret purposes come from the job, not invented ------------------ + + +def test_secrets_with_no_matching_job_purpose_are_dropped_and_logged( + tmp_path: Path, caplog: pytest.LogCaptureFixture, +) -> None: + """N10, p6-review-r2: every alias used to be relabelled `target_provider` unconditionally, + which silently defeats `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion (F13) the + moment such an alias ever reaches `secrets.json`. An alias with no matching `job.json`/ + `secret_purpose_map` entry must be DROPPED (never injected under a guessed purpose), not + silently relabelled — and the drop must be logged so a real injection gap is diagnosable.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123", "UNCLAIMED_ALIAS": "xyz"})) + envs_by_argv0: dict[tuple[str, ...], dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, # UNCLAIMED_ALIAS has no entry. + ) + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + agent_env = envs_by_argv0[("python3", "agent.py")] + assert agent_env.get("LIVEKIT_API_KEY") == "abc123" # the matched alias still lands. + assert "UNCLAIMED_ALIAS" not in agent_env # the unmatched one is never injected anywhere. + assert any("UNCLAIMED_ALIAS" in record.message for record in caplog.records) + + +def test_read_job_secret_purposes_reads_agent_secret_refs_from_job_json(tmp_path: Path) -> None: + """N10, p6-review-r2: the REAL hosted-path source — `/work/job.json`'s own `agent.secret_refs` + (§1: `{alias: {manager, key, version, purpose}}`) — not just the constructor override.""" + (tmp_path / "job.json").write_text(json.dumps({ + "agent": {"secret_refs": { + "LIVEKIT_API_KEY": { + "manager": "platform-vault", "key": "k", "version": None, + "purpose": "target_provider", + }, + }}, + })) + purposes = pr._read_job_secret_purposes(tmp_path) + assert purposes == {"LIVEKIT_API_KEY": "target_provider"} + + +def test_read_job_secret_purposes_absent_file_returns_empty_and_logs( + tmp_path: Path, caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + purposes = pr._read_job_secret_purposes(tmp_path) + assert purposes == {} + assert any("job.json" in record.message for record in caplog.records) + + +def test_load_and_delete_secrets_malformed_job_json_is_typed_not_attribute_error( + tmp_path: Path, +) -> None: + """N10, p6-review-r2: `read_text`/`json.loads` failures and a non-dict payload used to escape + `provision()` untyped (N9's own class of gap) — a non-dict `job.json` used to raise + `AttributeError` from `raw.items()` deep inside, not the typed failure this pins.""" + (tmp_path / "job.json").write_text(json.dumps(["not", "a", "dict"])) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._read_job_secret_purposes(tmp_path) + assert excinfo.value.code == "spawn_failed" + + +def test_load_and_delete_secrets_malformed_secrets_json_is_typed(tmp_path: Path) -> None: + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text("not valid json{{{") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._load_and_delete_secrets(secrets_path, work_directory=tmp_path) + assert excinfo.value.code == "spawn_failed" + + +# --- N2, p6-review-r2 (BLOCKER): promote-path polling --------------------------------------------- + + +def _manifest_with_fast_readiness_poll(*, timeout_seconds: float = 0.3) -> EnvironmentBundleV2: + """A tiny declared readiness timeout/interval so the poll tests below run in milliseconds, not + up to `_DEFAULT_STORE_READY_TIMEOUT_SECONDS`'s 30s floor.""" + return _manifest( + lambda body: {**body, "readiness": [ + {**body["readiness"][0], "timeout_seconds": timeout_seconds, "interval_seconds": 0.01}, + ]} + ) + + +def _recording_selective_prober(*, protocol: CapabilityProtocol, fail_times: int) -> Any: + """Like `_recording_prober`, but only ever fails for calls matching `protocol` — every other + protocol passes immediately. Isolates which call site actually needed to retry, since a bare + `_recording_prober` shared across the whole `SpawnContext` would also be consumed by `_wait_ + for_store_ready`'s own postgres readiness check at freeze time.""" + calls: list[dict[str, Any]] = [] + remaining = [fail_times] + + def prober(**kwargs: Any) -> bool: + calls.append(kwargs) + if kwargs.get("protocol") is protocol and remaining[0] > 0: + remaining[0] -= 1 + return False + return True + + prober.calls = calls # type: ignore[attr-defined] + return prober + + +def test_provision_promotion_polls_the_declared_probe_rather_than_sampling_once( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): `provision()`'s PREPARING->READY promotion used to call + `probe_runtime_health` exactly once, immediately, with no wait — nothing waits for the DAG's + TERMINAL process (`spawn_world` only waits on `depends_on` edges), so a bundle whose + readiness-bearing process is terminal was marked `UNHEALTHY` on every provision regardless of + how quickly it actually came up. A prober that fails twice before passing proves the + promotion actually POLLS. + + Q3, p6-review-r3: `_manifest_with_terminal_readiness`, not `_manifest_with_fast_readiness_ + poll` — the latter kept the `tools` readiness entry, backed by `tools-api`, which `agent`'s + own `depends_on` wait also polls during `spawn_world` and silently ate the scripted failures + BEFORE the promote-time poll ever ran, so `len(http_calls) >= 3` passed even with a reverted, + single-sample `_poll_runtime_health` (verified empirically). The terminal-readiness manifest's + only probe is backed by `agent` itself, which nothing `depends_on` — every HTTP call recorded + is unambiguously the promote-site poll.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.3) + source, bundle_dir = _provision_dirs(tmp_path) + prober = _recording_selective_prober(protocol=CapabilityProtocol.HTTP, fail_times=2) + provider = _sql_spy_provider(prober=prober, secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.READY + http_calls = [c for c in prober.calls if c.get("protocol") is CapabilityProtocol.HTTP] + assert len(http_calls) >= 3 # 2 failures + the passing call. + + +def test_reset_promotion_polls_the_declared_probe_rather_than_sampling_once( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): same fix, the OTHER promote site — `reset()`'s post-sentinel + probe. `template_database` postgres skips `_wait_for_store_ready` entirely on reset (the + job-shared engine is already running). + + Q3, p6-review-r3: `_manifest_with_terminal_readiness`, same reasoning as the provision-side + test above — its only readiness probe is backed by `agent`, the DAG's terminal process, which + nothing `depends_on`, so nothing else on the reset path can consume the flaky prober's failure + budget before the promotion poll itself runs.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.3) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + flaky = _recording_selective_prober(protocol=CapabilityProtocol.HTTP, fail_times=2) + provider._context = dc_replace(provider._context, prober=flaky) + + asyncio.run(provider.reset(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.READY + http_calls = [c for c in flaky.calls if c.get("protocol") is CapabilityProtocol.HTTP] + assert len(http_calls) >= 3 + + +def _manifest_with_terminal_readiness(*, timeout_seconds: float = 0.05) -> EnvironmentBundleV2: + """§2a's own documented shape: a readiness-bearing capability backed by the DAG's TERMINAL + process (`agent` — nothing in `_manifest()`'s topology ever names it in a `depends_on`). + `wait_for_dependency` never checks it during build; only `probe_runtime_health`'s promote-time + sweep of `manifest.readiness` does. The pre-existing `tools` entry is dropped so `tools-api`'s + own depends_on wait (from `agent`) has nothing to poll and returns immediately — build must + succeed regardless of what the promote-time prober does.""" + return _manifest( + lambda body: {**body, "readiness": [ + {"capability": "control", "path": "/health", "timeout_seconds": timeout_seconds, + "interval_seconds": 0.01}, + ], "capabilities": { + **body["capabilities"], + "control": {"protocol": "http", "service": "agent", "configuration_name": None}, + }} + ) + + +def test_provision_promotion_exhausts_the_poll_and_returns_unhealthy_never_raises( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): exhaustion is a state DECISION, not a provisioning failure — + `provision()` must return the world `UNHEALTHY`, never raise, when the declared probe never + passes within its own timeout — pinned against the exact §2a "terminal process" shape N2 + itself names.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.05) + source, bundle_dir = _provision_dirs(tmp_path) + + def always_fails_http(**kwargs: Any) -> bool: + return kwargs.get("protocol") is not CapabilityProtocol.HTTP + + provider = _sql_spy_provider( + prober=always_fails_http, secrets_path=tmp_path / "secrets.json", + ) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.UNHEALTHY + + +# --- N3, p6-review-r2 (MAJOR): a failed first provision() must not poison the provider ----------- + + +def test_provision_failed_first_freeze_resets_identity_so_a_retry_retries_the_build( + tmp_path: Path, +) -> None: + """N3, p6-review-r2: the job identity (`_manifest`/`_bundle_digest`) used to be committed + BEFORE `freeze_baseline` ran — a failure there left the instance claiming an identity with no + build output, so an in-process retry with the SAME bundle took the RECONCILE branch and + raised `internal_invariant_violated` instead of retrying the build and surfacing the real, + often-retryable cause. A `sql_runner` that raises only on its FIRST call (freeze's own `CREATE + DATABASE` for the template) then behaves normally proves the retry re-attempts the build.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + call_count = [0] + real_spy = SqlSpy() + + def flaky_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + call_count[0] += 1 + if call_count[0] == 1: + raise RuntimeError("connection reset by peer") + return real_spy(**kwargs) + + provider = _sql_spy_provider( + sql_runner=flaky_sql_runner, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "store_statement_failed" # the REAL cause, not a bogus invariant. + assert provider._manifest is None # identity reset, not left claiming a half-built job. + + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.READY + + +# --- N4, p6-review-r2 (MAJOR): no spawn path may orphan a live engine on a partial failure -------- + + +def _manifest_with_two_postgres_stores() -> EnvironmentBundleV2: + return _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "postgres2", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "database2": { + "protocol": "postgres", "service": "postgres2", + "configuration_name": "DATABASE2_URL", + }, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "database2", "migrations": [], "seed_files": [], + "baseline": {"strategy": "template_database", "inputs_digest": "sha256:" + "e" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1"}}, + ]}} + ) + + +def test_freeze_baseline_terminates_an_earlier_stores_handle_on_a_later_stores_failure( + tmp_path: Path, +) -> None: + """N4, p6-review-r2 (MAJOR): a raise partway through `freeze_baseline`'s per-store loop used + to leave every EARLIER store's own job-shared engine live and referenced NOWHERE — `self. + _job_shared_handles` is only ever assigned once this function RETURNS. Store 1 (`postgres`) + succeeds and is sealed `template_database`; store 2 (`postgres2`) fails its own seed — store + 1's handle must be terminated, not orphaned holding its formula port. + """ + manifest = _manifest_with_two_postgres_stores() + real_spy = SqlSpy() + + def flaky_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + statement = kwargs["statement"] + if statement.startswith("CREATE DATABASE") and "alk_baseline_postgres2" in statement: + raise RuntimeError("simulated store-2 seed failure") + return real_spy(**kwargs) + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=flaky_sql_runner, work_directory=tmp_path, + ) + handles: list[Any] = [] + original_runner = ctx.runner + + def recording_runner(*args: Any, **kwargs: Any) -> Any: + handle = original_runner(*args, **kwargs) + handles.append(handle) + return handle + + ctx = dc_replace(ctx, runner=recording_runner) + + with pytest.raises(pr.ProcessRuntimeError): + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert len(handles) == 2 # both postgres/postgres2 were actually spawned. + assert all(handle.terminated for handle in handles) # neither orphaned. + + +def test_ensure_world_publishes_partial_handles_so_close_can_terminate_them( + tmp_path: Path, +) -> None: + """N4, p6-review-r2 (MAJOR): a mid-clone failure (here: `agent`'s own `depends_on` wait on + `tools-api` timing out) used to drop `tools-api`'s already-spawned handle on the floor — `self. + _world_handles[world_index]` was only ever assigned on `_ensure_world`'s SUCCESS path. + `close()` (or the next reconcile) must be able to terminate it instead of orphaning it. + """ + manifest = _manifest_with_fast_readiness_poll(timeout_seconds=0.05) + source, bundle_dir = _provision_dirs(tmp_path) + + def failing_http_prober(**kwargs: Any) -> bool: + return kwargs.get("protocol") is not CapabilityProtocol.HTTP + + provider = _sql_spy_provider( + prober=failing_http_prober, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "depends_on_timeout" + tools_handle = provider._world_handles[0]["tools-api"] + assert tools_handle.handle.terminated is False # not yet — close() has not run. + + asyncio.run(provider.close(work_directory=tmp_path)) + assert tools_handle.handle.terminated is True # N4: reachable, and now cleaned up. + + +# --- N9, p6-review-r2 (MAJOR): filesystem failures must not escape untyped ----------------------- + + +def test_spawn_managed_process_data_dir_setup_failure_is_typed_spawn_failed(tmp_path: Path) -> None: + """N9, p6-review-r2: §4.6 — filesystem failures during provisioning are `infrastructure`. + mkdir/chmod/chown for a (re)spawned engine's own data directory used to raise bare, giving + `provision()`'s caller nothing to map.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + + def failing_chown(path: Path, uid: int, gid: int) -> None: + raise PermissionError("simulated: svc-control cannot chown this path") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_managed_process( + postgres, port=14000, data_dir=tmp_path / "pg", credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=failing_chown, + ) + assert excinfo.value.code == "spawn_failed" + assert excinfo.value.stage == "spawn" + + +def test_run_conformance_gate_never_raises_on_a_filesystem_fault_during_reset( + tmp_path: Path, +) -> None: + """N9, p6-review-r2: M3's own "TRULY never raises" promise used to hold only for + `ProcessRuntimeError` — `reset_world` beneath the gate runs `rmtree`/`copytree` (`_seal_world_ + store`'s own `datadir_copy` work), and a filesystem fault there must still degrade the gate, + never crash the job.""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + + def failing_copy(src: Path, dst: Path) -> None: + raise OSError("simulated: disk full mid-copy") + + broken_ctx = dc_replace(ctx, copy=failing_copy) + passed, reason = pr.run_conformance_gate( + manifest, context=broken_ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +# --- N5/N6, p6-review-r2 (MAJOR): rabbitmq.conf carries credentials; env var matches on disk ------ + + +def _rabbitmq_process() -> Any: + return pr.ManagedProcess.model_validate({ + "name": "queue", "kind": "managed", "engine": "rabbitmq", "version": "3.13", + "user": "svc-data", "depends_on": [], + }) + + +def test_spawn_managed_process_rabbitmq_conf_carries_credentials( + tmp_path: Path, +) -> None: + """N5, p6-review-r2 (MAJOR): a BARE `rabbitmq-server` (no Docker, §0) reads `default_user`/ + `default_pass` from the generated `rabbitmq.conf`, not from `RABBITMQ_DEFAULT_USER`/ + `RABBITMQ_DEFAULT_PASS` (a Docker-entrypoint-only convention) — without this the node + initializes with the built-in `guest` account and every rabbitmq call this module makes + (harness-authenticated) 401s. Q4, p6-review-r3: no `loopback_users` assertion — that line + widened `guest`'s reach instead of narrowing it and was dropped from the generated conf.""" + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + data_dir = tmp_path / "queue" + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + ) + conf_path = data_dir / "rabbitmq.conf" + conf_text = conf_path.read_text(encoding="utf-8") + assert "default_user = harness" in conf_text + assert "default_pass = s3cr3t" in conf_text + assert "loopback_users" not in conf_text + assert (conf_path.stat().st_mode & 0o777) == 0o600 # carries the password in cleartext. + + +def test_spawn_managed_process_rabbitmq_config_file_env_matches_the_on_disk_filename_exactly( + tmp_path: Path, +) -> None: + """N6, p6-review-r2 (MAJOR): `RABBITMQ_CONFIG_FILE` must carry the FULL path, extension + included — the pre-3.7 "the server appends .conf itself" behavior this used to depend on is + not how the catalog's 3.13 works, and the failure is silent (falls back to the default + management port, 15672 — inside §2b's own per-world port band).""" + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "queue" + captured_env: dict[str, str] = {} + + def recording_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured_env.update(env) + return FakeHandle() + + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=recording_runner, sync_run=_fake_sync_run, + ) + conf_path = data_dir / "rabbitmq.conf" + assert captured_env["RABBITMQ_CONFIG_FILE"] == str(conf_path) + assert Path(captured_env["RABBITMQ_CONFIG_FILE"]).name == "rabbitmq.conf" + assert conf_path.exists() + + +def test_spawn_managed_process_rabbitmq_conf_overwrites_a_datadir_copy_style_pre_populated_file( + tmp_path: Path, +) -> None: + """N5, p6-review-r2: a `datadir_copy` restore legitimately copies a PRIOR `rabbitmq.conf` in + before this function ever runs (M8's own note) — the write must overwrite it (fresh + credentials/port every spawn), never fail `FileExistsError` the way an `O_EXCL` create would.""" + data_dir = tmp_path / "queue" + data_dir.mkdir(parents=True) + (data_dir / "rabbitmq.conf").write_text("stale content from a copied baseline\n") + creds = pr.EngineCredentials(username="harness", password="fresh-pw") + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + ) + conf_text = (data_dir / "rabbitmq.conf").read_text(encoding="utf-8") + assert "fresh-pw" in conf_text + assert "stale content" not in conf_text + + +# --- Q2, p6-review-r3 (MAJOR): rabbitmq's mnesia data path must not depend on the node name ------- + + +def test_rabbitmq_daemon_env_mnesia_dir_is_node_name_free(tmp_path: Path) -> None: + """Q2, p6-review-r3 (MAJOR): the port-derived `RABBITMQ_NODENAME` must not leak into the mnesia + DATA path — `RABBITMQ_MNESIA_DIR` is a fixed path under `data_dir`, independent of port/node + name. R2, p6-review-r4: `RABBITMQ_MNESIA_BASE` stays set alongside it (rabbitmq derives OTHER + paths — e.g. `RABBITMQ_PLUGINS_EXPAND_DIR` — from the base, not the dir), so every path-valued + key the env names, `MNESIA_BASE` included, must still resolve under `data_dir`.""" + creds = pr.EngineCredentials(username="harness", password="pw") + env = pr.rabbitmq_daemon_env(data_dir=tmp_path, port=15003, credentials=creds) + assert env["RABBITMQ_MNESIA_DIR"] == str(tmp_path / "mnesia") + assert "15003" not in env["RABBITMQ_MNESIA_DIR"] # not derived from the port/node name. + for key in ("RABBITMQ_MNESIA_DIR", "RABBITMQ_MNESIA_BASE", "RABBITMQ_LOG_BASE"): + assert Path(env[key]).is_relative_to(tmp_path), f"{key} escapes data_dir: {env[key]}" + + +def test_rabbitmq_daemon_env_names_the_same_relative_mnesia_path_in_every_world( + tmp_path: Path, +) -> None: + """Q2, p6-review-r3 (MAJOR): world 0's bootstrap and world 1's restored instance boot on + DIFFERENT ports (rabbitmq is never job-shared, §2b), yet both name `/ + mnesia` for `RABBITMQ_MNESIA_DIR` — the port/node name no longer changes which relative path + the daemon looks under. This is necessary, not sufficient, for the copied baseline to actually + load past world 0 — see p6-review-r4 R1 (mnesia's on-disk schema is still node-name-bound).""" + manifest = _manifest_with_rabbitmq_store() + envs: list[dict[str, str]] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + envs.append(env) + return FakeHandle() + + ctx = _spawn_context(manifest, bundle_dir=tmp_path, runner=runner, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + bootstrap_env = next(e for e in envs if "RABBITMQ_MNESIA_DIR" in e) + + envs.clear() + pr._clone_or_reset_world( + manifest, 1, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + world1_env = next(e for e in envs if "RABBITMQ_MNESIA_DIR" in e) + + bootstrap_data_dir = tmp_path / "managed" / "queue" + world1_data_dir = tmp_path / "worlds" / "w1" / "queue" + assert bootstrap_env["RABBITMQ_MNESIA_DIR"] == str(bootstrap_data_dir / "mnesia") + assert world1_env["RABBITMQ_MNESIA_DIR"] == str(world1_data_dir / "mnesia") + # different ports still mean different node names (epmd registration) — only the DATA path, + # not the identity, had to stop depending on it. + assert world1_env["RABBITMQ_NODENAME"] != bootstrap_env["RABBITMQ_NODENAME"] + + +# --- N7, p6-review-r2 (MAJOR): rabbitmq readiness must probe BOTH listeners ----------------------- + + +def test_wait_for_store_ready_probes_both_amqp_and_management_listeners_for_rabbitmq( + tmp_path: Path, +) -> None: + """N7, p6-review-r2: the AMQP listener comes up during CORE boot; the management plugin's HTTP + listener comes up in the PLUGIN boot step that follows — the same B4 race, unfixed until now + for the one engine whose seed/sentinel/canary statements all travel over the listener that was + never probed.""" + manifest = _manifest_with_rabbitmq_store() + process = next(p for p in manifest.processes if p.name == "queue") + probed: list[tuple[Any, int, str | None]] = [] + + def recording_prober(**kwargs: Any) -> bool: + probed.append((kwargs["protocol"], kwargs["port"], kwargs.get("path"))) + return True + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=recording_prober, work_directory=tmp_path, + ) + port = ctx.port_plan.port_for("queue", 0) + pr._wait_for_store_ready(manifest, process, port=port, context=ctx) + + seen = {(protocol, p) for protocol, p, _ in probed} + assert (pr.CapabilityProtocol.AMQP, port) in seen + management_port = pr._rabbitmq_management_port(port) + assert (CapabilityProtocol.HTTP, management_port) in seen + http_paths = [path for protocol, _, path in probed if protocol is CapabilityProtocol.HTTP] + assert "/api/overview" in http_paths + + +# --- N11, p6-review-r2 (MAJOR): a dead job-shared engine must be respawned + every world resealed - + + +def test_reset_respawns_a_dead_job_shared_engine_and_reseals_every_tracked_world( + tmp_path: Path, +) -> None: + """N11, p6-review-r2 (MAJOR): a job-shared `template_database` engine (postgres, in + `_manifest()`) that dies mid-job used to be carried forward as if healthy — every subsequent + `reset()`/reconcile then raised `store_statement_failed` against a data directory sitting + right there on disk. Respawned from that surviving data directory, then EVERY world this + provider currently tracks (not just the one being reset) has its logical DB re-sealed from the + template — `reset()` is only called for world 0 here; world 1's own reseal can only have come + from the respawn's own multi-world loop. + """ + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + sql_spy = SqlSpy() + provider = _sql_spy_provider(sql_runner=sql_spy, secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + old_shared = provider._job_shared_handles["postgres"] + old_shared.handle.terminated = True # simulate the shared postgres dying (e.g. OOM-killed). + sql_spy.calls.clear() + + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + + new_shared = provider._job_shared_handles["postgres"] + assert new_shared is not old_shared # respawned, not silently reused dead. + statements = [statement for _, statement in sql_spy.calls] + assert 'CREATE DATABASE "w0" TEMPLATE "alk_baseline_postgres"' in statements + assert 'CREATE DATABASE "w1" TEMPLATE "alk_baseline_postgres"' in statements # never the + # `reset()` TARGET — only reachable via the respawn's own reseal-every-tracked-world loop. + assert runtimes[0].state is pr.RuntimeState.READY + + +def test_dead_job_shared_engine_respawn_failure_is_typed_naming_the_engine(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + old_shared = provider._job_shared_handles["postgres"] + old_shared.handle.terminated = True + + def failing_runner(argv, *, cwd, env, log_path, user=None, group=None): + raise OSError("no such device") + + provider._context = dc_replace(provider._context, runner=failing_runner) + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + assert excinfo.value.code == "store_statement_failed" + assert excinfo.value.process == "postgres" + + +# --- N12/N13, p6-review-r2 (MINOR): termination order, SIGINT for postgres, kill escalation ------- + + +def test_close_terminates_in_reverse_topological_order(tmp_path: Path) -> None: + """N12, p6-review-r2: dependency-first termination (`spawn_world`'s own insertion order) sends + SIGTERM to a per-world engine while its own dependents may still hold open connections, + guaranteeing the full escalation wait every time. Reversed so dependents (`agent`, `tools-api`) + are torn down BEFORE the engine they depend on (`postgres`).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + terminate_order: list[str] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + handle = FakeHandle() + name = Path(cwd).name + + def recording_terminate(_name: str = name, _handle: FakeHandle = handle) -> None: + terminate_order.append(_name) + _handle.terminated = True + + handle.terminate = recording_terminate # type: ignore[method-assign] + return handle + + provider = _sql_spy_provider(runner=runner, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + terminate_order.clear() + asyncio.run(provider.close(work_directory=tmp_path)) + per_world_order = [name for name in terminate_order if name in ("tools-api", "agent")] + assert per_world_order == ["agent", "tools-api"] + + +def test_close_prefers_sigint_over_sigterm_for_postgres(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + interrupted: list[str] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + handle = FakeHandle() + name = Path(cwd).name + + def recording_interrupt(_name: str = name, _handle: FakeHandle = handle) -> None: + interrupted.append(_name) + _handle.interrupted = True + _handle.terminated = True + + handle.interrupt = recording_interrupt # type: ignore[method-assign] + return handle + + provider = _sql_spy_provider(runner=runner, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + asyncio.run(provider.close(work_directory=tmp_path)) + assert "postgres" in interrupted # SIGINT (fast shutdown), never SIGTERM, for postgres. + + +def test_terminate_and_wait_escalates_to_kill_when_the_process_ignores_terminate() -> None: + """N13, p6-review-r2: `FakeHandle.wait` used to report the fake as exited the instant + `terminate()` was called, so no test ever reached the `kill()` branch — M7's escalation half + had zero coverage. A handle that ignores `terminate()`/`interrupt()` entirely (`wait()` only + reports exited once `kill()` has actually run) forces the real escalation path.""" + + class StubbornHandle: + def __init__(self) -> None: + self.terminated = False + self.killed = False + + def is_running(self) -> bool: + return not self.killed + + def captured_output(self) -> str: + return "" + + def terminate(self) -> None: + self.terminated = True # acknowledges the signal but does NOT actually exit. + + def interrupt(self) -> None: + self.terminated = True + + def wait(self, timeout: float) -> bool: + return self.killed + + def kill(self) -> None: + self.killed = True + + handle = StubbornHandle() + pr._terminate_and_wait(handle, timeout=0.01) + assert handle.terminated is True + assert handle.killed is True # escalation actually happened. + assert handle.is_running() is False # reaped. + + +def test_terminate_and_wait_logs_when_the_process_survives_even_kill( + caplog: pytest.LogCaptureFixture, +) -> None: + """N13, p6-review-r2: the post-`kill()` `wait()` return used to be discarded — a child that + STILL had not exited after SIGKILL (a wedged kernel wait, nothing left to escalate to) left no + trace anywhere. Now logged.""" + + class UnkillableHandle: + def is_running(self) -> bool: + return True + + def captured_output(self) -> str: + return "" + + def terminate(self) -> None: + pass + + def interrupt(self) -> None: + pass + + def wait(self, timeout: float) -> bool: + return False # never reports exited, even after kill() below. + + def kill(self) -> None: + pass + + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + pr._terminate_and_wait(UnkillableHandle(), timeout=0.01) + assert any("did not exit even after kill" in record.message for record in caplog.records) + + +# --- N14, p6-review-r2 (MINOR): chmod before chown ------------------------------------------------- + + +def test_spawn_managed_process_chmods_before_chowning_the_data_dir(tmp_path: Path) -> None: + """N14, p6-review-r2: `chmod` after `chown` cannot succeed once ownership has moved — `svc- + control` still owns `data_dir` at chmod time and it is free; after chown, a non-root `svc- + control` that can chown but not re-chmod a path it no longer owns would raise + `PermissionError`. Pinned by recording the directory's OWN mode at the moment `chown` fires.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + mode_at_chown_time: list[int] = [] + + def recording_chown(path: Path, uid: int, gid: int) -> None: + if path == data_dir: + mode_at_chown_time.append(data_dir.stat().st_mode & 0o777) + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=recording_chown, + ) + assert mode_at_chown_time == [0o700] + + +# --- N16, p6-review-r2 (MINOR): terminate backends before dropping a world's shared logical DB ---- + + +def test_drop_world_shared_databases_terminates_backends_before_dropping(tmp_path: Path) -> None: + """N16, p6-review-r2: mirrors `_reset_template_database`'s own sibling call — a lingering + backend connection on the world's logical DB otherwise blocks this DROP exactly the way it + would block a reuse-time one, silently re-opening the space leak m4 closed.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + sql_spy = SqlSpy() + provider = _sql_spy_provider(sql_runner=sql_spy, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + sql_spy.calls.clear() + provider._drop_world_shared_databases(1) + statements = [statement for _, statement in sql_spy.calls] + assert len(statements) == 2 + assert statements[0].startswith("SELECT pg_terminate_backend") + assert statements[1] == 'DROP DATABASE IF EXISTS "w1"' + + +# --- healthy() port method (task 2c) --------------------------------------------------------------- + + +def test_provider_healthy_port_never_promotes_from_preparing_or_unhealthy(tmp_path: Path) -> None: + """Task 2c / p6-review-r2: unlike the module-level `healthy()`, the PORT method must never + promote from ANY state — not even `preparing`, which only `provision()`'s own poll (N2) + promotes out of.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + + runtime.state = pr.RuntimeState.PREPARING + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is True + assert runtime.state is pr.RuntimeState.PREPARING # never promoted, even on a pass. + + runtime.state = pr.RuntimeState.UNHEALTHY + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is True + assert runtime.state is pr.RuntimeState.UNHEALTHY # still never promoted. + + +def test_provider_healthy_port_demotes_ready_to_unhealthy_on_a_failing_probe( + tmp_path: Path, +) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + assert runtime.state is pr.RuntimeState.READY + + provider._prober = lambda **kwargs: False + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is False + assert runtime.state is pr.RuntimeState.UNHEALTHY + + +def test_provider_healthy_port_demotes_both_the_providers_record_and_the_callers_object( + tmp_path: Path, +) -> None: + """After N1 these are the SAME object, but the port method must not silently rely on that — + pinned by checking both references independently.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + provider._prober = lambda **kwargs: False + asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.UNHEALTHY + assert provider._runtimes[0].state is pr.RuntimeState.UNHEALTHY + assert provider._runtimes[0] is runtime # N1: one object per world for the provider's life. + + +def test_provider_healthy_port_raises_typed_before_provision(tmp_path: Path) -> None: + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + fake_runtime = pr.EnvironmentRuntime( + runtime_id="x", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.healthy(fake_runtime, work_directory=tmp_path)) + assert excinfo.value.code == "internal_invariant_violated" From 41319e1bd40407bdf113643762fced29c4de0fc5 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 17:27:58 +0530 Subject: [PATCH 09/20] fix(harness): guard the degrade write site and close the provisioner suite's mutation blind spots Signed-off-by: khushalsonawat --- src/fi/alk/harness/process_runtime.py | 15 +- tests/harness/test_process_runtime.py | 309 ++++++++++++++++++++++++++ 2 files changed, 321 insertions(+), 3 deletions(-) diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py index eb7fc5e7..092ee005 100644 --- a/src/fi/alk/harness/process_runtime.py +++ b/src/fi/alk/harness/process_runtime.py @@ -202,7 +202,7 @@ def plan_ports(manifest: EnvironmentBundleV2, *, instances: int) -> PortPlan: job_shared=job_shared, fixed_ports=fixed_ports, effective_instances=1 if fixed_ports else instances, - degraded_reason="fixed_port" if fixed_ports else None, + degraded_reason="fixed_port" if fixed_ports and instances > 1 else None, ) @@ -3563,7 +3563,13 @@ def _provision_sync( ) requested = instances - degrade_reason = port_plan.degraded_reason # "fixed_port", or None (m1, p6-review-r1). + # At requested=1 nothing can degrade (effective=1 too, so no valid + # `parallelism_degraded` payload exists; outbound-channels.md's `1 <= effective < + # requested` bound is empty at requested=1). `port_plan.degraded_reason` is already + # None at instances=1, but that alone is not sufficient — see the + # `build_output.degrade_reason` write below, which is what actually enforces it + # against the conformance-gate paths that reassign `degrade_reason` further down. + degrade_reason = port_plan.degraded_reason if effective < requested else None if effective > 1 and not self._conformance_checked: self._ensure_world(0) @@ -3587,7 +3593,10 @@ def _provision_sync( build_output.requested_parallelism = requested build_output.effective_parallelism = effective - build_output.degrade_reason = degrade_reason + # The conformance-gate branches above reassign `degrade_reason` unconditionally (they + # only know a gate result, not this call's `requested`) — re-checking the invariant here, + # at the single write site, covers those paths too instead of only the fixed_port input. + build_output.degrade_reason = degrade_reason if effective < requested else None write_build_output(work_directory, build_output) # Reconcile down first — a prior call may have over-provisioned (the canary's own 2-world diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py index 60f752b1..ddc7aeb0 100644 --- a/tests/harness/test_process_runtime.py +++ b/tests/harness/test_process_runtime.py @@ -2167,6 +2167,26 @@ def failing_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: assert "syntax error" in str(excinfo.value) +def test_apply_seed_file_rabbitmq_import_failure_is_seed_failed(tmp_path: Path) -> None: + """rabbitmq's own definitions-import failure is customer-authored seed CONTENT + (§2f `seed_failed`, deterministic, NOT retried), never `store_statement_failed` (the + harness's own statement seam, `infrastructure`, retryable) — the two domains carry opposite + retry semantics, so a code swap here either retries a broken definitions file forever or + gives up on what might have been a transient import blip.""" + creds = pr.EngineCredentials(username="harness", password="pw") + + def failing_import(*, host: str, port: int, credentials: Any, file: Path) -> None: + raise RuntimeError("malformed definitions") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.apply_seed_file( + pr.ManagedEngine.RABBITMQ, tmp_path / "mq" / "defs.json", port=14002, dbname="", + credentials=creds, process_name="mq", sync_run=_fake_sync_run, + rabbitmq_import=failing_import, + ) + assert excinfo.value.code == "seed_failed" + + # --- §2c sentinel checking ------------------------------------------------------------------------ @@ -2236,6 +2256,55 @@ def test_a_sentinel_query_attempting_a_write_fails(tmp_path: Path) -> None: assert excinfo.value.code == "store_statement_failed" +def test_call_redis_driver_exception_is_store_statement_failed() -> None: + """`_call_redis` wraps a provisioner-ISSUED command (sentinel/canary probes — + never customer seed content, which goes through `apply_seed_file` instead) — a driver + exception there is the harness's own statement seam, §2f `store_statement_failed` + (`infrastructure`, retryable), never `seed_failed` (`environment`, NOT retried). Swapped, a + transient redis blip during a sentinel check would never retry.""" + def failing_redis_runner(*, host: str, port: int, command: Any) -> Any: + raise RuntimeError("connection reset") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_redis( + failing_redis_runner, stage="conformance", process_name="cache", + host="localhost", port=15003, command=["GET", "x"], + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_call_rabbitmq_driver_exception_is_store_statement_failed() -> None: + """`_call_rabbitmq` wraps a provisioner-ISSUED queue inspection (sentinel/canary probes — + never customer seed content) — a driver exception there is the harness's own statement seam, + §2f `store_statement_failed` (`infrastructure`, retryable), never `seed_failed` + (`environment`, NOT retried). Swapped, a transient rabbitmq blip during a sentinel check would + never retry.""" + def failing_inspector(*, host: str, port: int, credentials: Any, queue: str) -> int: + raise RuntimeError("connection reset") + + creds = pr.EngineCredentials(username="harness", password="pw") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_rabbitmq( + failing_inspector, stage="conformance", process_name="mq", + host="localhost", port=14002, credentials=creds, queue="canary", + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_call_rabbitmq_action_driver_exception_is_store_statement_failed() -> None: + """`_call_rabbitmq_action` wraps the write-side canary declare/publish call — same B5 typing + as `_call_rabbitmq`, against a store that has already passed readiness. A driver exception + there is also §2f `store_statement_failed`, never `seed_failed`.""" + def failing_action(**kwargs: Any) -> None: + raise RuntimeError("connection reset") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_rabbitmq_action( + failing_action, stage="conformance", process_name="mq", action="declare", + ) + assert excinfo.value.code == "store_statement_failed" + + def test_measure_postgres_row_counts_passes_read_only_true(tmp_path: Path) -> None: """N8, p6-review-r2: the baseline row-count reads (`pg_tables`, `COUNT(*)`) are the other read call site named in the fix — pinned directly rather than only indirectly through a @@ -2991,6 +3060,89 @@ def test_conformance_gate_is_vacuously_true_with_no_canary_store(tmp_path: Path) assert (passed, reason) == (True, None) +class _StickyCanarySqlSpy(SqlSpy): + """Simulates a RESET that runs (DROP/CREATE TEMPLATE are still recorded + normally, so the isolation probe's own w0-vs-w1 read is unaffected) but fails to actually + clear the conformance canary from the world it was planted in — `_alk_conformance` stays + `to_regclass`-visible in that one database no matter how many times it gets dropped and + recreated. Proves `_verify_canary_absent` is load-bearing, not redundant with the per-world + declared-sentinel check (`SELECT 1`), which never queries the canary table at all.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._sticky_dbname: str | None = None + + def __call__(self, **kwargs: Any) -> list[tuple[Any, ...]]: + statement = kwargs["statement"].strip() + if statement.startswith("CREATE TABLE") and "_alk_conformance" in statement: + self._sticky_dbname = kwargs["dbname"] + rows = super().__call__(**kwargs) + if statement.startswith("SELECT to_regclass") and kwargs["dbname"] == self._sticky_dbname: + return [(True,)] + return rows + + +def test_conformance_gate_fails_when_reset_leaves_the_canary_behind(tmp_path: Path) -> None: + """A reset that (for whatever reason) fails to actually clear the reserved + `_alk_conformance` object must fail the gate via `_verify_canary_absent` — this is the + project's own named CRITICAL calibration example, "vacuous canary pass" (severity-grading.md). + Isolation and both worlds' own declared sentinels pass; only the post-reset canary-absence + check catches it.""" + manifest = _manifest() + sql_spy = _StickyCanarySqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +class _BadResetSentinelSqlSpy(SqlSpy): + """The store's OWN declared sentinel (`SELECT 1`) answers correctly at freeze + time (queried against the baseline database) but wrong against either world database + (`w0`/`w1`) — simulating a reset whose reseal left the declared state broken, which + `reset_world`'s `_check_all_sentinels` is supposed to catch and the gate is supposed to + escalate via `sentinel_ok`, distinct from (and reached before) the canary-absence check.""" + + def __call__(self, **kwargs: Any) -> list[tuple[Any, ...]]: + if kwargs["statement"].strip() == "SELECT 1" and kwargs["dbname"] in ("w0", "w1"): + return [(0,)] + return super().__call__(**kwargs) + + +def test_conformance_gate_fails_when_a_worlds_own_sentinel_fails_after_reset( + tmp_path: Path, +) -> None: + """`sentinel_ok` from the gate's own per-world `reset_world` calls must actually + gate the result — a broken reset that fails the store's declared sentinel must degrade + parallelism, not be silently overridden by an otherwise-clean canary-absence check.""" + manifest = _manifest() + sql_spy = _BadResetSentinelSqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + # --- §4 provision / reset / close: ProcessRuntimeProvider ------------------------------------------- @@ -3040,6 +3192,59 @@ def test_provision_reconciles_to_exactly_w_ready_worlds(tmp_path: Path) -> None: assert build_output["degrade_reason"] is None +def test_provision_fixed_port_at_w1_records_no_degrade(tmp_path: Path) -> None: + """`fixed_port` forces `effective_instances=1` regardless of `instances` — + at `instances=1` that's not a degrade, since requested==effective already. A prior bug copied + `PortPlan.degraded_reason` verbatim onto `build.json` here, so a job that asked for W=1 and + got W=1 reported a parallelism degrade that never happened (crashes `hosted_entrypoint`'s + `parallelism_degraded` emission downstream, whose payload requires `effective < requested`).""" + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 1 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] is None + + +def test_provision_fixed_port_above_w1_records_degrade(tmp_path: Path) -> None: + """Companion to the test above: at `instances>1` the same `fixed_port` constraint IS a real + degrade (`effective=1 < requested=3`), so `build.json` must still report it.""" + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 3 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] == "fixed_port" + + def test_provision_reads_seed_files_from_bundle_dir_not_source(tmp_path: Path) -> None: """B1, p6-review-r1: §2c seed/migration paths are bundle-relative and must resolve against the VERIFIED bundle directory, never the untrusted checkout — a bundle declares `migrations: @@ -3072,6 +3277,54 @@ def test_provision_reads_seed_files_from_bundle_dir_not_source(tmp_path: Path) - assert applied_file == str(bundle_dir / "db" / "schema.sql") +def test_provision_empty_strategy_first_clone_reads_seed_files_from_bundle_dir_not_work_directory( + tmp_path: Path, +) -> None: + """Mirrors the test above for the OTHER `apply_store_seed` call site — `_seal_world_store`'s + `empty`-strategy branch, reached on every world clone/reset, not just `freeze_baseline`'s + once-per-job seed. `strategy: empty` never seeds at freeze (§5.3's own no-op capture; postgres + itself does not support it, so a redis `cache` store — the same shape the pre-existing + empty-strategy fixtures use — isolates this second call site cleanly). Every existing + `empty`-strategy fixture used `bundle_dir == work_directory`, so a regression swapping the two + there was invisible — genuinely distinct directories here, same proof technique as the + freeze-path test above. A decoy file of the SAME NAME is planted under `work_directory` too, + with different content — asserting content, not just presence, is what actually proves the + code read from `bundle_dir` rather than merely finding a same-named file wherever it looked.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + source, bundle_dir = _provision_dirs(tmp_path) + (bundle_dir / "cache").mkdir(parents=True) + (bundle_dir / "cache" / "seed.txt").write_text("SET greeting hi\n") + (tmp_path / "cache").mkdir(parents=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting DECOY\n") + + calls: list[Any] = [] + provider = _sql_spy_provider( + sync_run=_recording_sync_run(calls), secrets_path=tmp_path / "secrets.json", + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + redis_calls = [(argv, kwargs) for argv, kwargs in calls if argv and "redis-cli" in argv] + assert redis_calls, "expected a redis-cli seed invocation from the empty-strategy RESET path" + assert redis_calls[0][1]["input"] == "SET greeting hi\n" + + def test_provision_is_idempotent_for_the_same_job_identity(tmp_path: Path) -> None: manifest = _manifest() source, bundle_dir = _provision_dirs(tmp_path) @@ -3159,6 +3412,37 @@ def test_provision_conformance_degrade_persists_across_a_later_reconcile_call( assert build_output["degrade_reason"] == "conformance_gate_failed" # m1 +def test_provision_reconcile_at_w1_after_gate_failure_records_no_degrade(tmp_path: Path) -> None: + """A sick-world recovery re-call can legitimately pass a smaller `instances` than the job's + original request (the module's own comment above the reconcile branch). The sticky + conformance-degrade branch does not know the current call's `requested` — at `instances=1`, + `effective == requested == 1` already, so re-surfacing the earlier gate failure there would + write `build.json` a `parallelism_degraded`-shaped record with no valid payload + (`effective < requested` is required, and 1 < 1 is false).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + sql_runner=SqlSpy(canary_leaks=True), secrets_path=tmp_path / "secrets.json", + ) + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in first] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["degrade_reason"] == "conformance_gate_failed" + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert [r.world_index for r in second] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 1 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] is None + + def test_provision_tears_down_before_rebuilding_on_a_bundle_digest_change(tmp_path: Path) -> None: """M6, p6-review-r1: a bundle-digest change (a re-sealed bundle mid-attempt) used to reassign this instance's own identity straight over the PREVIOUS job's still-running processes and @@ -3296,6 +3580,31 @@ def test_close_is_idempotent_and_removes_secrets_and_data_directories(tmp_path: asyncio.run(provider.close(work_directory=tmp_path)) # must not raise the second time. +def test_close_removes_a_secrets_file_left_behind_by_a_failed_load(tmp_path: Path) -> None: + """`_load_and_delete_secrets` only unlinks `secrets.json` AFTER `json.loads` succeeds — a + malformed file raises `secrets/spawn_failed` with the file still on disk. Every OTHER test + that reaches `close()` does so via a successful load, where the file is already gone by the + time `close()` runs, so its own unlink line is otherwise a no-op across the whole suite; + plaintext secrets surviving on the sandbox filesystem after a failed provision is exactly what + that unlink exists to prevent.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text("{not valid json") + provider = _sql_spy_provider(secrets_path=secrets_path) + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "spawn_failed" + assert secrets_path.exists() # the failed load raised before its own unlink ran. + + asyncio.run(provider.close(work_directory=tmp_path)) + assert not secrets_path.exists() + + # --- B3/§0.3, p6-review-r1: secrets lifetime and injection through provision() ----------------- From 3b730820022a8669a2188e51a57bc46a0cfe7eac Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 18:02:47 +0530 Subject: [PATCH 10/20] fix(harness): pin the capabilities fixture expiry far-future Signed-off-by: khushalsonawat --- tests/harness/test_outbound.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py index ba734b4a..cfd62c22 100644 --- a/tests/harness/test_outbound.py +++ b/tests/harness/test_outbound.py @@ -254,7 +254,9 @@ def test_whole_object_digest_rejects_a_non_string_dict_key() -> None: "attempt_id": _ATTEMPT_ID, "attempt_number": 1, "fence": "opaque-fence", - "expires_at": "2026-08-25T12:00:00.000Z", + # Far future: load_capabilities rejects an expired token against wall-clock + # time, so a near-term timestamp turns the whole fixture into a time bomb. + "expires_at": "2099-01-01T00:00:00.000Z", "token": "bearer-token", "endpoints": { "events": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/events/", @@ -367,7 +369,7 @@ def test_load_capabilities_rejects_a_non_https_endpoint(tmp_path: Path, channel: def test_load_capabilities_rejects_an_expired_token(tmp_path: Path) -> None: path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) with pytest.raises(CapabilitiesError) as excinfo: - load_capabilities(path, now=lambda: datetime(2027, 1, 1, tzinfo=timezone.utc)) + load_capabilities(path, now=lambda: datetime(2100, 1, 1, tzinfo=timezone.utc)) assert excinfo.value.code == "capabilities_expired" assert path.exists() # a failed load must never unlink -- same rule as any other rejection @@ -2016,7 +2018,7 @@ def _capabilities() -> HostedCapabilities: "attempt_id": "a1", "attempt_number": 1, "fence": "fence1", - "expires_at": "2026-08-25T12:00:00.000Z", + "expires_at": "2099-01-01T00:00:00.000Z", "token": "tok", "endpoints": ENDPOINTS, } From c1d82ad450a8b1c69a3e9602722bbb421c83975c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 19:10:54 +0530 Subject: [PATCH 11/20] =?UTF-8?q?fix(harness):=20scheduler=20hardening=20?= =?UTF-8?q?=E2=80=94=20typed=20failure=20pass-through,=20fence=20latch,=20?= =?UTF-8?q?close=20serialization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_scheduler.py | 416 +++++++++-- tests/harness/test_hosted_scheduler.py | 936 ++++++++++++++++++++++++- 2 files changed, 1267 insertions(+), 85 deletions(-) diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 3991723c..5d4672e1 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -12,10 +12,14 @@ `RuntimeState`). `WorldProvisioner` below is a structural `Protocol` matching `ProcessRuntimeProvider`'s actual async shape so tests can inject a fake without touching a real filesystem/subprocess tree. -- `outbound.py` is being written in parallel and its surface is not pinned yet, so nothing here - imports it. `OutboundPort` is this module's own minimal sink for the events/receipts it - produces, typed against `outbound-channels.md`'s closed vocabulary; whoever wires the real - client adapts to it. +- `OutboundPort` is this module's own minimal sink for the events/receipts it produces, typed + against `outbound-channels.md`'s closed vocabulary; whoever wires the real client adapts to it. + `outbound.py` is now committed and quiescent, so this module imports exactly three of its + exception types — `HostedFencedError`/`HostedChannelFailedError`/`HostedAttemptSupersededError`, + the full `ChannelState` "stop emitting" latch for one attempt — to recognize the one outbound + failure class that is NOT best-effort (a 401/403 fence, an exhausted channel, or a superseded + attempt must stop the run, not be logged and forgotten); nothing else from that module is + imported here. - The Scenario Generation Contract (Karthik, in review) is not available here either, so `Scenario` is this module's own minimal Protocol for what the loop needs: a key/id pair, `setup`/`ready`, and named sub-goal checks. Same for the simulated "call" itself (a different track's seam) — @@ -37,7 +41,8 @@ from typing import Any, Awaitable, Callable, Protocol, Sequence from .job import FailureDomain, HarnessStage -from .process_runtime import EnvironmentRuntime, RuntimeState +from .outbound import HostedAttemptSupersededError, HostedChannelFailedError, HostedFencedError +from .process_runtime import EnvironmentRuntime, ProcessRuntimeError, RuntimeState from .world.errors import ( WorldError, WorldQueryRejected, @@ -299,6 +304,42 @@ async def receipt(self, receipt: ResultReceipt) -> None: ... } _RETRYABLE_CODES = frozenset({"evidence_missing"}) +# hosted-execution-seams.md v1.13 §5.4/§2f: the closed provisioner build/run failure-code table -- +# these used to be discarded at the reset()/provision() seam (caught as a bare `Exception`, only +# `str()` surviving into `world_unhealthy.cause`), so a deterministic `environment`/`agent` fault +# (never retried) was re-reported as `world_pool_exhausted`/infrastructure and burned every +# whole-job retry on a failure that repeats identically. `spawn_failed` is contractually split +# managed->infrastructure / source->agent, but this module deliberately never reads `bundle` (see +# the module docstring) so it cannot tell which process kind failed at this seam -- conservatively +# `infrastructure` (matches today's behavior on the source-process half; the correct split is an +# open contract question, not resolved here). +_SECTION_2F_DOMAIN: dict[str, FailureDomain] = { + "source_tree_unavailable": FailureDomain.ENVIRONMENT, + "build_failed": FailureDomain.AGENT, + "runtime_unsupported": FailureDomain.ENVIRONMENT, + "spawn_failed": FailureDomain.INFRASTRUCTURE, + "depends_on_timeout": FailureDomain.INFRASTRUCTURE, + "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, + "seed_failed": FailureDomain.ENVIRONMENT, + "store_statement_failed": FailureDomain.INFRASTRUCTURE, +} +# v1.13: only these two domains are "never retried" -- a uniform §2f code across every unhealthy +# world in one of them surfaces as that code+domain; anything else (mixed codes, or any +# infrastructure-domain fault) stays `world_pool_exhausted` exactly as before. +_SECTION_2F_NEVER_RETRIED = frozenset({FailureDomain.ENVIRONMENT, FailureDomain.AGENT}) + +# The one `OutboundPort` failure class that is NOT best-effort -- outbound-channels.md: +# 401/403 -> "stop emitting, exit code 3 ... never an infra retry," and the same latch also covers +# a 404-exhausted channel and a 409 attempt-supersession (outbound.py's `ChannelState`: "a fence in +# substance"). Letting `_emit`/`_log`/`mark_unhealthy` swallow any of these the same way they +# swallow a transport hiccup ran a superseded attempt's entire scenario set after the platform had +# already fenced or superseded it. +_FATAL_OUTBOUND: tuple[type[Exception], ...] = ( + HostedFencedError, + HostedChannelFailedError, + HostedAttemptSupersededError, +) + # M13: an exception/overrun outcome leaves the world half-applied — world-handle-interface.md's # return-conventions rule is "the world is discarded and re-provisioned (a half-applied world is # never reused)." `ready_not_ready` is deliberately excluded: a precondition failing on the shared @@ -550,18 +591,29 @@ class NoWorldsAvailable(RuntimeError): """Every provisioned world is down and none is currently recoverable (spine v1.12 §5.4: "if ready worlds reach 0 the job FAILS in stage running, domain infrastructure" — declared only after in-flight re-provisioning completes without restoring a world), OR the pool has - been closed (R5: `reason="closed"`).""" + been closed (R5: `reason="closed"`). + + v1.13 §5.4: `code`/`domain` carry a uniform §2f never-retried code when every unhealthy + world's last re-provision attempt agreed on one — `None` (the default) means the caller falls + back to the generic `world_pool_exhausted`/infrastructure abort, exactly as before.""" - def __init__(self, message: str, *, reason: str = "exhausted") -> None: + def __init__( + self, + message: str, + *, + reason: str = "exhausted", + code: str | None = None, + domain: FailureDomain | None = None, + ) -> None: super().__init__(message) self.reason = reason + self.code = code + self.domain = domain _RECONCILE_MAX_ATTEMPTS = 3 _RECONCILE_BACKOFF_SECONDS = (0.05, 0.1) _LEASE_POLL_INTERVAL_SECONDS = 0.02 -_CLOSE_RECONCILE_WAIT_SECONDS = 30.0 # R4: bounded wait for an in-flight reconcile before close() -# falls back to cancelling it (which cannot stop a thread-backed provider call already running). class WorldPool: @@ -603,6 +655,11 @@ def __init__( self._down: set[int] = set() self._fresh: set[int] = set() # m9: provisioned/recovered but never yet leased/reset self._effective_size = 0 # R2: the achieved world count `start()` settled on + # The §2f code (or `None`) behind the most recent demotion/reconcile-failure for a down + # world index -- read by `lease()`'s exhaustion check to decide whether a uniform + # never-retried code can surface instead of the generic `world_pool_exhausted`. + self._down_codes: dict[int, str | None] = {} + self._fenced: BaseException | None = None # latched by mark_fenced(), never cleared # m1: `asyncio.Condition` (not a manual `Event` + `clear()`) — waiting and notifying share # one lock, so there is no window between releasing a lock and clearing a flag for a @@ -614,8 +671,11 @@ def __init__( self._started = False self._closing = False # R4: set at the top of close() -- lets an in-flight reconcile bail # between attempts instead of burning close()'s wait budget on a pool being torn down. - self._closed = False # R5: set once close() has actually run -- latches provision()/lease() - # out for good; close() itself becomes idempotent. + self._closed = False # Set once close() STARTS -- latches provision()/lease() out for + # good immediately, independent of whether teardown itself has finished. + self._teardown_task: asyncio.Task[None] | None = None # the shared, retry-safe + # teardown -- see close()'s own comment for why idempotency lives here now, not on + # `_closed`. @property def effective_size(self) -> int: @@ -625,6 +685,19 @@ def effective_size(self) -> int: off this, never off the originally requested `instances`.""" return self._effective_size + @property + def fenced(self) -> BaseException | None: + """The first fatal `OutboundPort` exception (401/403 -> `HostedFencedError`, a + 404-exhausted channel -> `HostedChannelFailedError`, or a 409 attempt-supersession -> + `HostedAttemptSupersededError`) observed anywhere along this pool's own emit paths. + `HostedScheduler` polls this at the same points it polls `cancel_requested` to stop + leasing/launching further scenarios once set.""" + return self._fenced + + def mark_fenced(self, exc: BaseException) -> None: + if self._fenced is None: + self._fenced = exc + @property def size(self) -> int: return len(self._runtimes) @@ -728,9 +801,28 @@ async def lease( if self._reconcile_in_flight() or self._reconcile_pending: await self._wait_bounded(poll=abandon is not None) continue + # v1.13 §5.4: a uniform §2f never-retried code across every + # currently-unhealthy world surfaces AS that code+domain; mixed codes, an + # unrecorded (non-§2f) cause, or any infrastructure-domain code all fall + # back to the generic `world_pool_exhausted` exactly as before. The + # uniformity set is `self._down` -- every unhealthy world in the pool, not + # just `usable` (runtimes minus this call's `exclude`) -- a world excluded + # because it is the scenario's own just-failed world is still part of "every + # unhealthy world" the spec means; narrowing to `usable` would let that + # excluded world's own (possibly untyped) failure escape the check entirely. + codes = {self._down_codes.get(index) for index in self._down} + code = domain = None + if len(codes) == 1: + (only_code,) = codes + if only_code is not None: + only_domain = _SECTION_2F_DOMAIN.get(only_code) + if only_domain in _SECTION_2F_NEVER_RETRIED: + code, domain = only_code, only_domain raise NoWorldsAvailable( f"{len(self._down)}/{len(self._runtimes)} worlds unhealthy, " - f"none available outside {sorted(exclude)}" + f"none available outside {sorted(exclude)}", + code=code, + domain=domain, ) await self._wait_bounded(poll=abandon is not None) continue @@ -739,6 +831,12 @@ async def lease( probed_runtime: EnvironmentRuntime | None = None if not skip_reset: async with self._provider_lock: + if self._closed: + # close() can win the `_provider_lock` FIFO queue against a lease already + # past the top-of-loop `_closed` check -- re-check on the inside too, or + # this lease drives reset() against a provider close() may already be + # hard-cleaning. + raise NoWorldsAvailable("world pool is closed", reason="closed") runtime = self._runtimes.get(world_index) probed_runtime = runtime if runtime is not None: @@ -754,6 +852,8 @@ async def lease( # R13 (spine v1.12 §4.5b): `healthy` now rides the port's non-reentrancy rule too, # so it goes under `_provider_lock` like reset/provision/close. async with self._provider_lock: + if self._closed: + raise NoWorldsAvailable("world pool is closed", reason="closed") # same re-check as above runtime = self._runtimes.get(world_index) probed_runtime = runtime if runtime is not None: @@ -774,6 +874,12 @@ async def lease( runtime = self._runtimes.get(world_index) if runtime is None or runtime is not probed_runtime: self._leased.discard(world_index) + if runtime is not None: + # The object was REPLACED, not removed -- put the index back where the + # outer loop can find it, or it lands nowhere (not available, not down) + # and every future lease() spins forever on a candidate set that never + # grows. + self._available.add(world_index) self._state_lock.notify_all() continue if is_healthy and runtime.state is RuntimeState.READY: @@ -784,8 +890,16 @@ async def lease( if reset_exc is not None else f"reset left world in state {runtime.state.value}" ) + # Preserve a typed §2f code across this seam instead of flattening it to free + # text -- `mark_unhealthy` records it so a later exhaustion declaration can tell a + # deterministic never-retried fault apart from a generic infrastructure one. + code = ( + reset_exc.code + if isinstance(reset_exc, ProcessRuntimeError) and reset_exc.code in _SECTION_2F_DOMAIN + else None + ) - await self.mark_unhealthy(world_index, cause=cause) + await self.mark_unhealthy(world_index, cause=cause, code=code) # loop again — this index is now excluded via `_down`, no explicit retry bookkeeping. async def release(self, world_index: int) -> None: @@ -795,12 +909,16 @@ async def release(self, world_index: int) -> None: self._available.add(world_index) self._state_lock.notify_all() - async def mark_unhealthy(self, world_index: int, *, cause: str) -> None: + async def mark_unhealthy(self, world_index: int, *, cause: str, code: str | None = None) -> None: async with self._state_lock: self._leased.discard(world_index) self._available.discard(world_index) self._fresh.discard(world_index) self._down.add(world_index) + # Unconditional -- every demotion overwrites the recorded reason (or clears a stale + # §2f code with `None` when this one isn't typed), so exhaustion always reads the + # MOST RECENT cause for this index, never a leftover from an earlier failure. + self._down_codes[world_index] = code runtime = self._runtimes.get(world_index) if runtime is not None: # M12 (spine v1.12 §4.5b, normative): the scheduler demotes `state` on the @@ -813,16 +931,21 @@ async def mark_unhealthy(self, world_index: int, *, cause: str) -> None: self._reconcile_pending = True self._state_lock.notify_all() + # Schedule recovery BEFORE the telemetry emit below -- `OutboundPort` calls are + # best-effort and may be slow or hang, and recovery must never sit behind one (worst + # case: `_reconcile_pending` stays latched and `lease()`'s grace loop spins forever). + await self._schedule_reconcile() + # R6: this is the sole path every demotion (this method) goes through, so it is the one # place `world_unhealthy` needs to be emitted from for all four call sites to get it. if self._outbound is not None: try: await self._outbound.world_unhealthy(world_index=world_index, cause=_sanitize_cause(cause)) + except _FATAL_OUTBOUND as exc: # a fence stops the run -- never best-effort. + self.mark_fenced(exc) except Exception as exc: # noqa: BLE001 - B3: outbound failures are never fatal. await self._log(f"world_unhealthy emit failed: {exc}") - await self._schedule_reconcile() - async def _schedule_reconcile(self) -> None: async with self._state_lock: if self._closed: @@ -876,10 +999,24 @@ async def _reconcile(self) -> None: break if last_exc is not None or runtimes is None: + # The FINAL failed re-provision attempt's typed §2f code, applied to every world + # still down when this reconcile gives up -- one `provision()` call covers the whole + # pool, so a typed failure here is uniform by construction across everything it did + # not just recover. + code = ( + last_exc.code + if isinstance(last_exc, ProcessRuntimeError) and last_exc.code in _SECTION_2F_DOMAIN + else None + ) # R8: every success path below ends in `notify_all()` — this give-up path must too, # or a `lease()` blocked in `_wait_bounded(poll=False)` (the `abandon is None` case) # waits forever for a reconcile that already gave up. + # Unconditional, mirroring `mark_unhealthy`'s own invariant -- an untyped final + # attempt must overwrite (clear) a stale typed code left by an earlier demotion, or + # exhaustion later reads that leftover code as if it were this attempt's own result. async with self._state_lock: + for index in self._down: + self._down_codes[index] = code self._state_lock.notify_all() return # stays `_down`; the next `mark_unhealthy` (or a lease-triggered wait) retries. @@ -888,15 +1025,32 @@ async def _reconcile(self) -> None: # here would be reading our own signal as independent proof. R13 (spine v1.12 §4.5b): # `healthy` now rides the port's non-reentrancy rule, so these probes go under # `_provider_lock` too. + if self._closing: + # provision() just succeeded, but close() may already be queued on `_provider_lock` + # for its own `provisioner.close()` call -- bail before racing it for one more round + # of provider calls the pool is being torn down under anyway. + return healthy_by_index: dict[int, bool] = {} + # The probe's own §2f code, carried alongside its verdict -- a world that comes back from a + # SUCCESSFUL `provision()` but fails this probe never enters the give-up path above (that + # path only fires on a raised/failed `provision()`), so without this the state block below + # has no code of its own and would otherwise leave whatever an earlier, superseded demotion + # recorded standing. + healthy_codes: dict[int, str | None] = {} async with self._provider_lock: for runtime in runtimes: try: healthy_by_index[runtime.world_index] = await self._provisioner.healthy( runtime, work_directory=self._work_directory ) - except Exception: # noqa: BLE001 + healthy_codes[runtime.world_index] = None + except Exception as exc: # noqa: BLE001 healthy_by_index[runtime.world_index] = False + healthy_codes[runtime.world_index] = ( + exc.code + if isinstance(exc, ProcessRuntimeError) and exc.code in _SECTION_2F_DOMAIN + else None + ) achieved = {runtime.world_index for runtime in runtimes} async with self._state_lock: @@ -905,10 +1059,15 @@ async def _reconcile(self) -> None: if healthy_by_index.get(runtime.world_index, False): was_down = runtime.world_index in self._down self._down.discard(runtime.world_index) + self._down_codes.pop(runtime.world_index, None) # recovered -- stale now if runtime.world_index not in self._leased: self._available.add(runtime.world_index) if was_down and runtime.state is RuntimeState.READY: self._fresh.add(runtime.world_index) # m9 + elif runtime.world_index in self._down: + # Still down after a successful re-provision -- this probe's own result + # replaces whatever an earlier demotion left, never a leftover from before it. + self._down_codes[runtime.world_index] = healthy_codes.get(runtime.world_index) # `provision` reconciles to exactly `instances` worlds (a conformance-gate degrade can # shrink `achieved` below what this pool started with) — anything no longer returned # is gone, not merely unhealthy. @@ -917,36 +1076,64 @@ async def _reconcile(self) -> None: self._available.discard(stale) self._down.discard(stale) self._fresh.discard(stale) + self._down_codes.pop(stale, None) # the index itself is gone # m3: NOT `_leased.discard(stale)` — an in-flight scenario may still hold this # index's lease (e.g. a conformance degrade shrinking `achieved` mid-scenario); # dropping the lease record here would make its later `release()`/ # `mark_unhealthy()` a silent no-op. Those methods already guard on # `world_index in self._runtimes`, so leaving `_leased` alone and letting them # reconcile it lazily is correct. + # Keep this truthful across a reconcile, not just at start() -- P10 sizes + # `parallelism_degraded` off it, and a reconcile can grow the pool back up or shrink + # it further (a conformance degrade narrowing `achieved`) in either direction. + self._effective_size = len(self._runtimes) self._state_lock.notify_all() async def close(self) -> None: async with self._state_lock: - if self._closed: - return # R5: idempotent, matching spine §4 point 4 ("close is idempotent"). - self._closed = True - self._closing = True + if not self._closed: + self._closed = True + self._closing = True + # Wake anything blocked in `lease()`'s `_wait_bounded(poll=False)` so it + # re-checks `_closed` instead of waiting for a recovery that will never come. + self._state_lock.notify_all() + # The OLD idempotency check (`if self._closed: return`) latched here, before teardown + # ever ran -- a caller wrapping this whole call in its own timeout (the entrypoint's + # `_bounded_close`) could cancel it mid-teardown, and a RETRY then hit that early + # return and silently never called `provisioner.close()` at all. `_closed` still has + # to latch immediately (lease()'s top-of-loop check, its inner re-check under + # `_provider_lock`, and `_teardown`'s own bail-out below all depend on new + # leases/reconciles being rejected the moment close() STARTS, not once it finishes), + # so idempotency now lives on a separate, SHARED teardown task instead: every call — + # first or retried — creates it once and awaits the same one. + if self._teardown_task is None: + self._teardown_task = asyncio.create_task(self._teardown()) + teardown_task = self._teardown_task + + # `asyncio.shield`: if THIS call's own awaiter is cancelled (the caller's timeout fires), + # the cancellation stops at this `await` and never reaches `teardown_task` -- teardown + # keeps running in the background, and a retry's `close()` re-attaches to the same + # (possibly by-then-finished) task instead of no-op'ing. + await asyncio.shield(teardown_task) + + async def _teardown(self) -> None: + async with self._state_lock: task = self._reconcile_task - # R5: wake anything blocked in `lease()`'s `_wait_bounded(poll=False)` so it re-checks - # `_closed` instead of waiting for a recovery that will never come. - self._state_lock.notify_all() - if task is not None: - # R4: `ProcessRuntimeProvider.provision`/`reset`/`healthy` are `asyncio.to_thread` — - # cancelling the awaiting coroutine does NOT stop the underlying thread, so - # cancelling immediately just races the hard-clean below against a `provision()` - # still repopulating `self._runtimes`/the worlds directory. Wait for the real work to - # finish on its own first; only cancel (accepting the thread may still leak, same - # bounded tradeoff as an abandoned scenario phase) if it blows the bound. - done, pending = await asyncio.wait({task}, timeout=_CLOSE_RECONCILE_WAIT_SECONDS) - for pending_task in pending: - pending_task.cancel() - await asyncio.gather(*done, *pending, return_exceptions=True) + # §4.5b: do NOT cancel-then-close. + # `ProcessRuntimeProvider.provision`/`reset`/`healthy` are `asyncio.to_thread` — + # cancelling the awaiting coroutine does NOT stop the underlying thread, so the old + # bounded-wait-then-cancel let `provisioner.close()` run CONCURRENTLY with a + # still-live `provision()` once the bound expired: unsynchronized identity dicts + # (`RuntimeError: dictionary changed size during iteration`), leaked engines, and + # `close()` itself could raise out of the guest's terminal path. `_closing` (in + # `_reconcile`'s own retry loop and healthy-probe gate) already makes a reconcile bail + # BETWEEN attempts/probes without a cancel, so this waits for the ONE call already in + # flight to finish on its own — unbounded from this function's perspective, but + # bounded in practice by whichever single `provision()`/`healthy()` call was running, + # with the outer flush-window deadline (spine, P10-owned) as the real backstop -- + # there is no longer a single constant here that bounds this wait on its own. + await asyncio.gather(task, return_exceptions=True) async with self._provider_lock: await self._provisioner.close(work_directory=self._work_directory) @@ -959,6 +1146,8 @@ async def _log(self, message: str, *, level: str = "error") -> None: # routinely carries a postgres error string with the DSN, and outbound-channels.md # requires redaction (no endpoint userinfo) before anything crosses the wire. await self._outbound.log(level=level, message=_sanitize_cause(message)) + except _FATAL_OUTBOUND as exc: # a fence stops the run -- never best-effort. + self.mark_fenced(exc) except Exception: # noqa: BLE001 - B3: outbound failures are never fatal. pass @@ -968,8 +1157,18 @@ async def _log(self, message: str, *, level: str = "error") -> None: @dataclass(frozen=True) class RunResult: + """`receipts` mixes already-emitted real receipts with synthesized-but-not-yet-emitted + `skipped` ones (R6) — see `HostedScheduler.emit_skipped_receipts`. + + `fenced` is set once a 401/403 (`HostedFencedError`), a 404-exhausted channel + (`HostedChannelFailedError`), or a 409 attempt-supersession (`HostedAttemptSupersededError`) + was observed on any outbound call -- the run stops launching further scenarios the moment it is + set. The caller maps this to exit code 3 and must not call `emit_skipped_receipts` (no further + outbound emission once fenced).""" + receipts: tuple[ResultReceipt, ...] aborted: ReceiptFailure | None + fenced: BaseException | None = None def _skipped_receipt(scenario: Scenario) -> ResultReceipt: @@ -1000,14 +1199,27 @@ def _unjudged(sub_goals: Sequence[SubGoal]) -> tuple[SubGoalResult, ...]: # phase threads that can ever be simultaneously abandoned (leaked) in one job. +def _abort_from_no_worlds(exc: NoWorldsAvailable) -> ReceiptFailure: + # A uniform §2f never-retried code across every unhealthy world surfaces AS that code+domain; + # otherwise this is the generic exhaustion abort. + if exc.code is not None and exc.domain is not None: + return ReceiptFailure( + domain=exc.domain.value, stage=HarnessStage.RUNNING.value, code=exc.code, message=_truncate(str(exc)) + ) + return _failure("world_pool_exhausted", str(exc)) + + @dataclass class _ScenarioContext: """R7: `_run_scenario` records the world/attempt it is currently working on here as it goes, so a crash that escapes every handled path still lets `worker()` report the REAL - world_index/scenario_attempt on the `driver_crashed` receipt instead of always None/1.""" + world_index/scenario_attempt on the `driver_crashed` receipt instead of always None/1. + `call` is set the moment the call step returns, so a LATER crash (e.g. `read_only()` + building the check-phase handle) still reports the call that genuinely ran, not `null`.""" world_index: int | None = None attempt: int = 1 + call: CallSummary | None = None @dataclass(frozen=True) @@ -1058,12 +1270,20 @@ async def run(self, scenarios: Sequence[Scenario]) -> RunResult: # thread at once (world-handle-interface.md: "its thread leaks, bounded by scenario # count"). self._executor = ThreadPoolExecutor( - max_workers=self._pool.effective_size + _LEAK_HEADROOM, thread_name_prefix="hosted-scenario" + # `_LEAK_HEADROOM` alone assumes spine §1's `scenario_count` admission cap (<=10) — + # widen for whatever `scenarios` actually holds, or an over-cap job's overflow + # scenarios find the executor saturated and report `driver_crashed` for a phase that + # was queued, not run. + max_workers=max(self._pool.effective_size + _LEAK_HEADROOM, len(scenarios) + 1), + thread_name_prefix="hosted-scenario", ) try: async def worker(index: int, scenario: Scenario) -> None: - if abort_holder[0] is not None or self._cancel_requested(): + # `self._pool.fenced` is the same stop-path as `abort_holder`/`cancel_requested` + # -- once any outbound call has hit a 401/403 or an exhausted channel, no further + # scenario may even start. + if abort_holder[0] is not None or self._pool.fenced is not None or self._cancel_requested(): return context = _ScenarioContext() try: @@ -1071,7 +1291,11 @@ async def worker(index: int, scenario: Scenario) -> None: scenario, index, abort_holder=abort_holder, context=context ) except NoWorldsAvailable as exc: - abort_holder[0] = _failure("world_pool_exhausted", str(exc)) + abort_holder[0] = _abort_from_no_worlds(exc) + except _FATAL_OUTBOUND: + # Already latched onto `self._pool.fenced` by whichever `_emit`/`_log` call + # raised it -- no receipt for a scenario the platform already superseded. + pass except asyncio.CancelledError: raise except BaseException as exc: # noqa: BLE001 @@ -1079,7 +1303,8 @@ async def worker(index: int, scenario: Scenario) -> None: # scenario's receipt — `gather(return_exceptions=True)` below is the second # half of that guarantee. results[index] = await self._driver_crashed_receipt( - scenario, exc, world_index=context.world_index, scenario_attempt=context.attempt + scenario, exc, world_index=context.world_index, scenario_attempt=context.attempt, + call=context.call, ) tasks = [asyncio.create_task(worker(i, s)) for i, s in enumerate(scenarios)] @@ -1090,10 +1315,14 @@ async def worker(index: int, scenario: Scenario) -> None: for index, scenario in enumerate(scenarios): receipt = results[index] if receipt is None: + # (outbound-channels.md v1.3 Sequencing: "terminal event -> skipped + # receipts -> manifest"): only SYNTHESIZE here. `run()` returns before its + # caller has emitted a terminal event, so pushing this over `outbound` now + # would put it on the wire ahead of the terminal -- `emit_skipped_receipts()` + # is the caller's job, done AFTER its own terminal event. receipt = _skipped_receipt(scenario) - await self._emit(self._outbound.receipt(receipt), what="receipt") receipts.append(receipt) - return RunResult(receipts=tuple(receipts), aborted=abort_holder[0]) + return RunResult(receipts=tuple(receipts), aborted=abort_holder[0], fenced=self._pool.fenced) finally: # R1: never block `run()` on abandoned threads — `shutdown(wait=True)` would hang # this coroutine exactly like the bug this fixes. Queued-but-unstarted work is @@ -1101,20 +1330,52 @@ async def worker(index: int, scenario: Scenario) -> None: # bounded tradeoff (world-handle-interface.md's "the job TTL is the backstop"). self._executor.shutdown(wait=False, cancel_futures=True) + async def emit_skipped_receipts(self, result: RunResult) -> None: + """R6: outbound-channels.md v1.3 Sequencing — "terminal event -> skipped receipts -> + manifest". `run()` only synthesizes `skipped` receipts into `RunResult.receipts`; call + this AFTER the caller's own terminal event has been emitted, never before, and exactly + once — each call re-emits every `skipped` receipt in `result.receipts` with no dedup of + its own. + + A no-op once `self._pool.fenced` is set (checked live, so it also covers a fence that + landed after `run()` returned but before this call) — the run stopped emitting the moment + the fence was observed and must not resume for these. If a fence instead lands DURING this + method's own loop, the same `_FATAL_OUTBOUND` that stops `run()` escapes out of this method + too; the caller must be ready for that.""" + if self._pool.fenced is not None: + return + for receipt in result.receipts: + if receipt.status == "skipped": + await self._emit(self._outbound.receipt(receipt), what="receipt") + async def _emit(self, awaitable: Awaitable[None], *, what: str) -> None: # B3: `OutboundPort` exceptions are best-effort telemetry — never receipt-affecting and # never fatal to the run. Logged through the same port when logging itself doesn't also # fail; swallowed otherwise rather than let a transport hiccup kill the scenario loop. + # The one exception besides `CancelledError` this deliberately does NOT swallow — a + # fence (401/403) or an exhausted channel (404x3) is never best-effort telemetry. try: await awaitable + except _FATAL_OUTBOUND as exc: + self._pool.mark_fenced(exc) + raise except Exception as exc: # noqa: BLE001 try: await self._outbound.log(level="error", message=f"outbound.{what} failed: {exc}") + except _FATAL_OUTBOUND as log_exc: + self._pool.mark_fenced(log_exc) + raise except Exception: # noqa: BLE001 pass async def _driver_crashed_receipt( - self, scenario: Scenario, exc: BaseException, *, world_index: int | None, scenario_attempt: int + self, + scenario: Scenario, + exc: BaseException, + *, + world_index: int | None, + scenario_attempt: int, + call: CallSummary | None = None, ) -> ResultReceipt: failure = _failure("driver_crashed", f"{type(exc).__name__}: {exc}") try: @@ -1132,7 +1393,7 @@ async def _driver_crashed_receipt( status="errored", sub_goals=sub_goals, evaluations=(), - call=None, + call=call, # the call step's own summary, if it had already returned when this crashed failure=failure, ) await self._emit(self._outbound.receipt(receipt), what="receipt") @@ -1160,7 +1421,9 @@ async def _lease_or_abandon( self, *, exclude: frozenset[int], abort_holder: list[ReceiptFailure | None] ) -> tuple[int, EnvironmentRuntime] | None: def _abandon() -> bool: - return abort_holder[0] is not None or self._cancel_requested() + # A scenario already queued in `lease()` must also abandon once fenced -- the + # worker-top check alone only stops scenarios that had not started yet. + return abort_holder[0] is not None or self._pool.fenced is not None or self._cancel_requested() return await self._pool.lease(exclude=exclude, abandon=_abandon) @@ -1186,6 +1449,8 @@ async def _run_scenario( context.world_index = world_index # R7: the real values for a driver_crashed receipt context.attempt = attempt + context.call = None # this attempt has not made its own call yet -- must not still + # carry a previous attempt's summary on the shared context object into this one's receipt. # B5: re-check immediately after `lease()` returns — a cancel/abort landing while this # worker was queued must not let a freshly granted world start work it can never finish @@ -1200,6 +1465,19 @@ async def _run_scenario( world_resolved = False # B3: the leased world must be released/discarded exactly once try: + if pending_retry is not None: + # Emitted here, immediately before attempt 2's own `scenario_started`, so this + # event and the pending-retry receipt (both exits above) are mutually exclusive + # by construction — outbound-channels.md Channel 2: "the failed first try is + # recorded by scenario_retried/world_unhealthy events, never by a receipt." + await self._emit( + self._outbound.scenario_retried( + scenario_key=scenario.scenario_key, + from_world=pending_retry.world_index, + to_world=world_index, + ), + what="scenario_retried", + ) await self._emit( self._outbound.scenario_started( scenario_key=scenario.scenario_key, @@ -1227,7 +1505,9 @@ async def _run_scenario( mark_unhealthy=True, ) else: - outcome = await self._execute(scenario, world, runtime, world_index, attempt=attempt) + outcome = await self._execute( + scenario, world, runtime, world_index, attempt=attempt, context=context + ) if isinstance(outcome, _Retry): # R6: `mark_unhealthy()` itself emits `world_unhealthy` now (every demotion path @@ -1264,7 +1544,7 @@ async def _run_scenario( # M8: this scenario already ran and produced a real attempt-1 failure — losing # it to skipped-synthesis just because the retry lease found nothing would # report "never ran" for a scenario that manifestly did. - abort_holder[0] = _failure("world_pool_exhausted", str(exc)) + abort_holder[0] = _abort_from_no_worlds(exc) # v1.13 §5.4 return await self._emit_pending_retry_receipt(scenario, pending) if next_leased is None: @@ -1272,12 +1552,6 @@ async def _run_scenario( # pool exhaustion. return await self._emit_pending_retry_receipt(scenario, pending) next_index, next_runtime = next_leased - await self._emit( - self._outbound.scenario_retried( - scenario_key=scenario.scenario_key, from_world=world_index, to_world=next_index - ), - what="scenario_retried", - ) return await self._run_scenario( scenario, scenario_index, @@ -1303,17 +1577,32 @@ async def _run_scenario( return outcome finally: if not world_resolved: - # B3: something blew past every handled path above (a bug in this module itself) - # — the world must not be silently stranded outside the pool's bookkeeping. - # Discarded rather than released: an exception here leaves its state unknown, and - # world-handle-interface.md's own exception rule is "discarded and re-provisioned, - # never reused." - await self._pool.mark_unhealthy( - world_index, cause="scenario driver crashed while holding this world" - ) + if self._pool.fenced is not None: + # The exception that skipped every path above was a fence (or the pool was + # already fenced by something else) -- the world itself never did anything + # wrong. `mark_unhealthy()` here would emit a false `world_unhealthy` after the + # run already stopped emitting, and schedule a `provision()` reconcile for a + # job that is not coming back for it. + await self._pool.release(world_index) + else: + # Something blew past every handled path above (a bug in this module + # itself) — the world must not be silently stranded outside the pool's + # bookkeeping. Discarded rather than released: an exception here leaves its + # state unknown, and world-handle-interface.md's own exception rule is + # "discarded and re-provisioned, never reused." + await self._pool.mark_unhealthy( + world_index, cause="scenario driver crashed while holding this world" + ) async def _execute( - self, scenario: Scenario, world: World, runtime: EnvironmentRuntime, world_index: int, *, attempt: int + self, + scenario: Scenario, + world: World, + runtime: EnvironmentRuntime, + world_index: int, + *, + attempt: int, + context: "_ScenarioContext", ) -> "ResultReceipt | _Retry": setup = await _run_phase( scenario.setup, world, timeout=SETUP_TIMEOUT_SECONDS, phase="setup", executor=self._executor @@ -1358,6 +1647,9 @@ async def _execute( sub_goals=_unjudged(scenario.sub_goals), call=None, ) + # Set the moment the call step returns, so a crash later in this method (e.g. + # `world.read_only()` below) still reports the call that genuinely ran, not `null`. + context.call = self._call_summary(call_outcome) calls = list(call_outcome.calls) # m12: `folder.py::_RUNNABLE` expects a list, not a tuple. if not calls: # M10: unconditioned on `turns` — an empty list must never reach checks regardless of diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index d5ede280..a21f1fa4 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -16,7 +16,14 @@ from typing import Any from fi.alk.harness import hosted_scheduler as hs -from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState +from fi.alk.harness.outbound import ( + ChannelError, + ChannelOutcome, + HostedAttemptSupersededError, + HostedChannelFailedError, + HostedFencedError, +) +from fi.alk.harness.process_runtime import EnvironmentRuntime, ProcessRuntimeError, RuntimeState from fi.alk.harness.world.errors import ( WorldReadOnly, WorldStateTooLarge, @@ -38,9 +45,15 @@ class FakeProvisioner: review named as fidelity gaps: `reset_scripts` lets a test script one world's next N reset outcomes (anything unscripted resets clean to READY), and every provider call — including `healthy()` (R13: spine v1.12 §4.5b folded it into the same non-reentrant set) — goes through - `_serialized`, which both yields (`await asyncio.sleep(0)` — so a genuine overlap has a real - chance to interleave) and asserts no second call is ever in flight at the same time, matching - "not reentrant" (B1/B2's own regression test).""" + `_serialized`, which yields (`await asyncio.sleep(0)` — so a genuine overlap has a real + chance to interleave) and records any overlapping call into `overlaps`, matching "not + reentrant" (B1/B2's own regression test). + + `_serialized` used to `assert not self._busy` in-band — but that assertion fires INSIDE + `_reconcile`'s own `except Exception` (a reconcile must never crash the pool) or `lease()`'s + `except Exception as exc: reset_exc = exc`, so production swallows it and the calling test + never sees a failure. Recording into `overlaps` and asserting on it from the test's OWN frame + is what actually makes a `_provider_lock` regression observable.""" def __init__( self, instances: int, *, reset_scripts: dict[int, list[RuntimeState]] | None = None @@ -49,28 +62,29 @@ def __init__( self.reset_scripts = reset_scripts or {} self.provision_calls = 0 self.reset_calls = 0 + self.healthy_calls = 0 self.closed = False self._runtimes = {i: _runtime(i) for i in range(instances)} - self._busy = False + self.overlaps: list[str] = [] + self._in_flight: list[str] = [] @contextlib.asynccontextmanager - async def _serialized(self): - # The `try/finally` wraps the yielding `sleep(0)` too — `close()` cancelling an in-flight - # reconcile (M6) must still clear `_busy`, or a cancellation lands this assertion stuck - # True forever and fails every later call in the same test for the wrong reason. - assert not self._busy, "provider port called reentrantly (provision/reset/healthy/close overlap)" - self._busy = True + async def _serialized(self, label: str): + if self._in_flight: + self.overlaps.append(f"{self._in_flight[-1]} overlapped {label}") + self._in_flight.append(label) try: await asyncio.sleep(0) yield finally: - self._busy = False + # `remove` (not `pop`) — cancellation (M6) can unwind these out of call order. + self._in_flight.remove(label) async def provision( self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, contract: Any | None = None, instances: int = 1, ) -> list[EnvironmentRuntime]: - async with self._serialized(): + async with self._serialized("provision"): self.provision_calls += 1 for index in range(instances): if index not in self._runtimes or self._runtimes[index].state in ( @@ -80,17 +94,18 @@ async def provision( return [self._runtimes[index] for index in range(instances) if index in self._runtimes] async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: - async with self._serialized(): + async with self._serialized(f"reset(w{runtime.world_index})"): self.reset_calls += 1 script = self.reset_scripts.get(runtime.world_index) runtime.state = script.pop(0) if script else RuntimeState.READY async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: - async with self._serialized(): + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 return runtime.state is RuntimeState.READY async def close(self, *, work_directory: Path) -> None: - async with self._serialized(): + async with self._serialized("close"): self.closed = True @@ -279,7 +294,7 @@ def test_a_freshly_provisioned_world_failing_its_health_probe_is_not_handed_out( async def scenario() -> None: class Provisioner(FakeProvisioner): async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: - async with self._serialized(): + async with self._serialized(f"healthy(w{runtime.world_index})"): return runtime.world_index != 0 pool, _ = _pool(2, provisioner=Provisioner(2)) @@ -364,7 +379,9 @@ async def close(self, *, work_directory): def test_concurrent_mark_unhealthy_never_calls_provision_reentrantly() -> None: # T7/B1: two worlds going bad in the same tick must serialize onto one provider call at a - # time — `FakeProvisioner._serialized`'s own assertion is what actually catches a regression. + # time. `overlaps` is asserted here, in the TEST's own frame — an in-band assert inside + # `_serialized` would instead land inside `_reconcile`'s `except Exception` and never fail + # this test. async def scenario() -> None: pool, provisioner = _pool(2) await pool.start() @@ -377,16 +394,22 @@ async def scenario() -> None: assert pool.size == 2 world_index, runtime = await pool.lease() assert runtime.state is RuntimeState.READY + assert provisioner.overlaps == [] await pool.close() + assert provisioner.overlaps == [] asyncio.run(scenario()) def test_lease_reset_and_a_background_reconcile_never_overlap_on_the_provider() -> None: - # TH-4/B2: the overlap that actually matters is a lease()'s reset() running concurrently with - # a DIFFERENT world's reconcile provision() -- the old coalescer test never drove two - # DIFFERENT provider calls at once; `FakeProvisioner._serialized`'s reentrancy assertion is - # what would catch a `_provider_lock` regression, so this drives it for real. + # TH-4/B2: the overlap that actually matters is a lease()'s reset() running concurrently + # with a DIFFERENT world's reconcile provision() -- the old coalescer test never drove two + # DIFFERENT provider calls at once. `overlaps` is asserted from the test's own frame: the + # old in-band assert inside `_serialized` fired inside `_reconcile`'s `except Exception` / + # `lease()`'s `except Exception as exc: reset_exc = exc` and was silently swallowed by + # production error-handling, so all three `_provider_lock` sites had zero effective + # coverage. `reset_calls` is pinned too, so the test cannot silently stop driving the + # overlap it claims to. async def scenario() -> None: pool, provisioner = _pool(2) await pool.start() @@ -408,6 +431,54 @@ async def scenario() -> None: assert leased is not None and leased[0] == 1 await asyncio.sleep(0.1) # let the reconcile finish assert provisioner.provision_calls >= 2 + assert provisioner.reset_calls >= 1 + assert provisioner.overlaps == [] + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_recovers_the_world_index_when_the_provider_call_races_a_replaced_runtime_object() -> None: + # The R14 discard branch used to drop a replaced-object's index from `_leased` without + # putting it back anywhere -- not `_available`, not `_down`. Every later candidate set then + # stays empty forever and `lease()` hangs. Dead in production today (the real provider + # mutates `EnvironmentRuntime` in place, never replaces it), but the fix has to hold + # independent of that reachability argument, so this drives the replacement directly. + async def scenario() -> None: + holder: dict[str, Any] = {} + + class SwapOnce(FakeProvisioner): + def __init__(self, instances: int) -> None: + super().__init__(instances) + self.armed = False # only swap once the test's OWN lease call is under way + self._swapped = False + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + if self.armed and not self._swapped: + self._swapped = True + # Simulate a concurrent reconcile replacing this index's own + # `EnvironmentRuntime` object between this lease's provider call and its + # post-call re-read. + holder["pool"]._runtimes[runtime.world_index] = _runtime( + runtime.world_index, RuntimeState.READY + ) + return runtime.state is RuntimeState.READY + + provisioner = SwapOnce(1) + pool, _ = _pool(1, provisioner=provisioner) + holder["pool"] = pool + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + + provisioner.armed = True + before = provisioner.healthy_calls + world_index, runtime = await asyncio.wait_for(pool.lease(), timeout=1.0) + assert world_index == 0 + assert runtime.state is RuntimeState.READY + assert provisioner.healthy_calls - before == 2 # the swapped attempt, then the retry that succeeded await pool.close() asyncio.run(scenario()) @@ -463,6 +534,53 @@ async def close(self, *, work_directory): asyncio.run(scenario()) +def test_close_during_an_in_flight_reconcile_never_overlaps_the_providers_close_call() -> None: + # §4.5b: the old bounded-wait-then-cancel let close() run `provisioner.close()` CONCURRENTLY + # with a still-live `provision()` once the internal 30s bound expired -- cancelling the + # awaiting coroutine cannot stop underlying thread-backed work. Real `asyncio.to_thread` + # dispatch here (a plain `asyncio.sleep`-backed fake would let even the OLD cancel-then-close + # behavior look correct, since cancellation genuinely stops a sleeping coroutine). `overlaps` + # -- the same state mechanism the reentrancy tests above build -- is the assertion a + # regression back to cancel-then-close would trip. + async def scenario() -> None: + release_provision = threading.Event() + + class SlowThreadedProvisioner(FakeProvisioner): + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + + def _blocking() -> list[EnvironmentRuntime]: + release_provision.wait(timeout=5.0) + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + + runtimes = await asyncio.to_thread(_blocking) + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + return runtimes + + provisioner = SlowThreadedProvisioner(1) + release_provision.set() + pool, _ = _pool(1, provisioner=provisioner) + await pool.start() + release_provision.clear() + + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="boom") # schedules a reconcile mid-flight + await asyncio.sleep(0.05) # let the reconcile's provision() actually begin on its thread + + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert not provisioner.closed, "close() ran the provider's own close() before provision() returned" + + release_provision.set() # let the thread-backed provision() finish on its own + await asyncio.wait_for(close_task, timeout=5.0) + assert provisioner.closed is True + assert provisioner.overlaps == [] + + asyncio.run(scenario()) + + def test_mark_unhealthy_and_lease_after_close_are_blocked() -> None: # R5: `close()` latches -- neither a late `mark_unhealthy()` (e.g. a scenario's `finally` # racing a SIGTERM-triggered close()) nor a fresh `lease()` may touch the provider again once @@ -492,6 +610,127 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_close_retried_after_a_callers_own_timeout_still_closes_the_provisioner() -> None: + # A caller wrapping the WHOLE close() call in its own timeout (the entrypoint's + # `_bounded_close`) used to cancel close() after `_closed` had already latched -- + # a retry then hit the old `if self._closed: return` idempotency check and returned + # IMMEDIATELY, claiming completion while teardown could still be running (or, with the check + # removed but no shield, cancelled the shared teardown outright). Either bug shows up as the + # retry NOT genuinely blocking until teardown finishes -- checked directly below, rather than + # via an eventually-true `provisioner.closed` (a to_thread-backed close() leaks its thread on + # real cancellation and can flip that flag on its own regardless of whether close() re-awaited + # it, the same non-cancellable-thread shape the test above covers). + async def scenario() -> None: + events: list[str] = [] + release_close = threading.Event() + + class SlowCloseProvisioner(FakeProvisioner): + async def close(self, *, work_directory: Path) -> None: + async with self._serialized("close"): + + def _blocking() -> None: + release_close.wait(timeout=5.0) + events.append("provider-close-done") + + await asyncio.to_thread(_blocking) + self.closed = True + + provisioner = SlowCloseProvisioner(1) + pool, _ = _pool(1, provisioner=provisioner) + await pool.start() + + # First call: the CALLER's own timeout fires while provisioner.close() is still blocked + # on its thread -- this must not stop the teardown itself (asyncio.shield). + try: + await asyncio.wait_for(pool.close(), timeout=0.1) + except asyncio.TimeoutError: + pass + else: + raise AssertionError("expected the first close() to time out while provisioner.close() blocks") + events.append("first-close-timed-out") + assert provisioner.closed is False # still mid-teardown, not abandoned + + # Retry, started while the provider is STILL blocked (release_close not yet set) -- it + # must genuinely wait, not return claiming completion (the early-return bug) or race an + # independently-cancelled teardown to a coincidentally-correct result. + retry_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert not retry_task.done(), "retry close() returned before teardown actually finished" + events.append("retry-still-waiting") + + release_close.set() + await asyncio.wait_for(retry_task, timeout=2.0) + events.append("retry-close-returned") + + assert events == [ + "first-close-timed-out", "retry-still-waiting", "provider-close-done", "retry-close-returned", + ] + assert provisioner.closed is True + assert provisioner.overlaps == [] + + asyncio.run(scenario()) + + +def test_close_blocks_a_reset_that_wins_the_provider_lock_race_after_close_has_latched() -> None: + # `_closed` is set (under `_state_lock`, no `_provider_lock` needed) the moment close() + # starts -- but a lease already past the top-of-loop `_closed` check can still + # win the `_provider_lock` FIFO queue race and call reset() against a provider close() is + # about to hard-clean. Reproduces the race by holding `_provider_lock` externally so both + # close() and a queued lease() are genuinely waiting on it when it releases. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + + await pool._provider_lock.acquire() # stand in for "some provider call already in flight" + lease_task = asyncio.create_task(pool.lease()) + await asyncio.sleep(0.05) # let lease() clear the top `_closed` check and queue on the lock + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) # close() latches `_closed` (no lock needed) and also queues + pool._provider_lock.release() # FIFO: lease() was queued first, so it goes first + + try: + await asyncio.wait_for(lease_task, timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + assert provisioner.reset_calls == 0 # never touched the provider once closed had latched + + await asyncio.wait_for(close_task, timeout=1.0) + + asyncio.run(scenario()) + + +def test_close_blocks_a_healthy_probe_that_wins_the_provider_lock_race_after_close_has_latched() -> None: + # The healthy() block specifically: a freshly-provisioned world skips reset() (m9) and + # goes straight to healthy() -- that site needs the same re-check as the reset block above, + # not just the reset block. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() # world 0 is "fresh" -- its first lease skips reset(), pays for healthy() + + await pool._provider_lock.acquire() + lease_task = asyncio.create_task(pool.lease()) + await asyncio.sleep(0.05) + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + pool._provider_lock.release() + + try: + await asyncio.wait_for(lease_task, timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + assert provisioner.healthy_calls == 0 + + await asyncio.wait_for(close_task, timeout=1.0) + + asyncio.run(scenario()) + + def test_start_degrades_when_provision_returns_fewer_worlds_than_instances() -> None: # R2: `provision()` legitimately returns fewer worlds than requested (conformance-gate # failure, `fixed_port`) — spine v1.12 §4: "Fail -> effective parallelism 1 ... @@ -521,6 +760,43 @@ async def close(self, *, work_directory): asyncio.run(scenario()) +def test_effective_size_tracks_a_reconcile_not_just_start() -> None: + # `_effective_size` used to be a start()-time snapshot `_reconcile` never touched -- P10 + # sizes `parallelism_degraded` off this property, so a stale value would announce the wrong + # world count outbound. Here start() degrades to 1 of 3, then a reconcile recovers the full + # 3 -- `effective_size` must follow it back up. + async def scenario() -> None: + calls = {"n": 0} + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return [_runtime(0, RuntimeState.READY)] # degraded: 1 of 3 requested + return [_runtime(i, RuntimeState.READY) for i in range(instances)] # fully recovered + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(3, provisioner=Provisioner()) + await pool.start() + assert pool.effective_size == 1 + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="force a reconcile") + await asyncio.sleep(0.1) + assert pool.size == 3 + assert pool.effective_size == 3 + await pool.close() + + asyncio.run(scenario()) + + def test_start_rejects_a_genuinely_malformed_provision_result() -> None: # R2: the degrade allowance is not a blanket exemption — zero worlds and a non-contiguous # index set are still rejected as malformed. @@ -572,6 +848,287 @@ async def gap() -> None: asyncio.run(gap()) +def test_pool_exhaustion_surfaces_a_uniform_never_retried_section_2f_code_from_lease() -> None: + # hosted-execution-seams.md v1.13 §5.4: a deterministic §2f fault (domain environment/agent, + # never retried) used to be discarded at the reset()/reconcile seam and re-reported as + # retryable `world_pool_exhausted`/infrastructure -- burning every whole-job retry on a fault + # that fails identically every time. + # + # Isolated to `lease()`'s OWN reset()/healthy() extraction: `provision()` always SUCCEEDS + # (so the reconcile's separate give-up extraction never runs and cannot backfill the same + # code), and the reconcile's post-provision healthy-probe failing again is what keeps the + # world down -- a first version of this test had `provision()` also fail typed, which masked + # a mutated-away `lease()` extraction because the reconcile's own give-up path recorded the + # same code independently. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + outbound = FakeOutbound() + pool, _ = _pool(1, provisioner=Provisioner(1), outbound=outbound) + await pool.start() + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1, + ) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is not None + assert result.aborted.code == "seed_failed" + assert result.aborted.domain == "environment" + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_surfaces_a_uniform_never_retried_section_2f_code_from_reconcile() -> None: + # The OTHER extraction site, isolated the same way in reverse -- `reset()` fails UNTYPED + # (so `lease()`'s own extraction always yields `None` and cannot backfill), and only the + # reconcile's give-up path ever sees a typed `ProcessRuntimeError`. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise RuntimeError("generic reset failure") # untyped -- lease()'s own code is None + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "seed_failed" + assert exc.domain is hs.FailureDomain.ENVIRONMENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_with_mixed_section_2f_codes_stays_world_pool_exhausted() -> None: + # Mixed codes across the unhealthy worlds must NOT surface either one -- v1.13 only + # promotes a code that is UNIFORM across every currently-unhealthy world. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + code = "seed_failed" if runtime.world_index == 0 else "store_statement_failed" + raise ProcessRuntimeError("reset", code, "boom") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + code = "seed_failed" if runtime.world_index == 0 else "store_statement_failed" + raise ProcessRuntimeError("reset", code, "boom") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + # Untyped -- must not overwrite `_down_codes` with a uniform code. + raise RuntimeError("provider is generically down") + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_with_a_uniform_infrastructure_domain_code_stays_world_pool_exhausted() -> None: + # `store_statement_failed` IS §2f-typed and uniform here, but its domain is + # infrastructure (retryable) -- v1.13 only promotes `environment`/`agent`, so this must still + # fall back to the generic exhaustion abort. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_give_up_with_an_untyped_final_attempt_clears_a_stale_typed_code() -> None: + # A world demoted by a typed `seed_failed` reset failure used to keep that code in + # `_down_codes` forever if the reconcile that follows gives up UNTYPED (a bare `OSError`, or + # any non-§2f exception) -- the give-up path only overwrote when its OWN failure was typed, + # so this stale code outlived the attempt that actually produced it and a later exhaustion + # declaration read it as if it were current. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + # Every reconcile attempt after the initial `start()` fails UNTYPED -- the + # give-up path must clear the `seed_failed` code the reset() failure recorded, + # not leave it standing. + raise OSError("transient enospc") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_exhaustion_considers_every_down_world_not_just_the_exclude_narrowed_subset() -> None: + # A retry lease's `exclude` set narrows `usable` to the runtimes NOT being avoided -- using + # `usable` (rather than every currently-unhealthy world) as the uniformity set let the + # excluded world's own failure escape the check entirely, so a lone untyped down world + # sitting outside `exclude` could hide behind a uniform typed code from everyone else and + # wrongly surface as that code+domain instead of the generic exhaustion abort. + # + # Isolated to the uniformity computation alone: `healthy()` always returns `False` (never + # raises), so the post-provision recovery branch never runs and never pops `_down_codes` -- + # only the two codes this test sets directly via `mark_unhealthy()` are ever in play. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return False + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + await pool.mark_unhealthy(0, cause="generic reset failure", code=None) # untyped + await pool.mark_unhealthy(1, cause="seed_failed", code="seed_failed") # typed, environment + try: + # Excludes world 0 -- a scenario retrying away from the world it just failed on. The + # only OTHER down world (1) carries a uniform typed code on its own, but world 0's own + # untyped failure must still block the pass-through. + await asyncio.wait_for(pool.lease(exclude=frozenset({0})), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_success_with_a_failed_health_probe_clears_a_stale_typed_code() -> None: + # A world demoted with a typed code can be re-provisioned successfully and still fail its + # post-provision health probe -- that probe returning `False` (not raising) is untyped, and + # the give-up path above never runs on this branch because `provision()` itself succeeded. The + # world stays down, but nothing about this round produced a §2f code, so a later exhaustion + # declaration must see the generic `world_pool_exhausted` shape, not the code an earlier, + # superseded demotion happened to record. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return False # untyped -- provision() itself always succeeds + + pool, provisioner = _pool(1, provisioner=Provisioner(1)) + await pool.start() + await pool.mark_unhealthy(0, cause="seed reset failed", code="seed_failed") # stale typed code + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + assert provisioner.provision_calls >= 2 # the reconcile actually re-provisioned + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_success_with_a_typed_health_probe_failure_surfaces_that_codes_own_domain() -> None: + # The other direction of the same probe: when the post-provision health check itself raises a + # typed §2f error, that is the round's own result and must replace whatever an earlier, + # superseded demotion recorded -- not merely clear it to `None`. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("healthy", "build_failed", "container image missing on retry") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + await pool.mark_unhealthy(0, cause="seed reset failed", code="seed_failed") # different, stale code + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "build_failed" + assert exc.domain is hs.FailureDomain.AGENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + def test_world_unhealthy_emitted_exactly_once_per_demotion_path() -> None: # R6: every demotion path now goes through `WorldPool.mark_unhealthy()`, the sole emitter — # parametrized over three distinct triggers (the fourth, the `_Retry(mark_unhealthy=True)` @@ -912,6 +1469,65 @@ def maybe_runaway(world: Any) -> None: asyncio.run(scenario()) +def test_scenario_phases_run_on_the_dedicated_hosted_scenario_executor() -> None: + # R1's actual claim is WHOSE executor phase threads run on, not merely how many workers it + # has -- reverting `loop.run_in_executor(executor, _run)` back to + # `asyncio.to_thread(_run)` (the loop's shared default executor) survived every test in the + # suite, because nothing pinned the dispatch target itself. Capture the real thread name from + # inside a phase body. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + seen_thread_name = {"name": ""} + + def capture(world: Any) -> None: + seen_thread_name["name"] = threading.current_thread().name + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=capture, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "passed" + assert seen_thread_name["name"].startswith("hosted-scenario"), seen_thread_name["name"] + await pool.close() + + asyncio.run(scenario()) + + +def test_executor_sizing_covers_more_scenarios_than_the_leak_headroom_alone() -> None: + # `_LEAK_HEADROOM` alone assumes spine §1's `scenario_count` admission cap (<=10) -- W=1, N + # scenarios that all abandon their setup thread, N > effective_size + _LEAK_HEADROOM (11). + # Before the fix, scenarios past the headroom found the executor saturated and were reported + # `driver_crashed` for a phase that never even started, instead of the genuine + # `setup_timeout` every one of them actually is. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.1 + try: + def runaway(world: Any) -> None: + time.sleep(5.0) # abandoned -- never returns within the test + + n = 13 # > effective_size(1) + _LEAK_HEADROOM(10) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario(f"s{i}", f"id-{i}", setup_fn=runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]) + for i in range(n) + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=15.0) + codes = [r.failure.code for r in result.receipts if r.failure is not None] + assert codes.count("driver_crashed") == 0, codes + assert codes.count("setup_timeout") == n + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await asyncio.wait_for(pool.close(), timeout=2.0) + + asyncio.run(scenario()) + + def test_check_broken_leaves_later_subgoals_unjudged() -> None: async def scenario() -> None: outbound = FakeOutbound() @@ -1007,6 +1623,39 @@ def blow_up(world: Any, calls: Any) -> None: asyncio.run(scenario()) +def test_mark_unhealthy_schedules_recovery_before_the_telemetry_emit() -> None: + # `_schedule_reconcile()` used to run AFTER the `world_unhealthy` emit -- a slow (or hanging) + # `OutboundPort` call delayed recovery by exactly its own duration, and worst case (an emit + # that never returns) `_reconcile_pending` stays latched forever and `lease()`'s grace loop + # spins without ever declaring exhaustion. + async def scenario() -> None: + emit_started = asyncio.Event() + emit_release = asyncio.Event() + + class SlowOutbound(FakeOutbound): + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + emit_started.set() + await emit_release.wait() + await super().world_unhealthy(world_index=world_index, cause=cause) + + outbound = SlowOutbound() + pool, provisioner = _pool(1, outbound=outbound) + await pool.start() + world_index, _ = await pool.lease() + + mark_task = asyncio.create_task(pool.mark_unhealthy(world_index, cause="boom")) + await asyncio.wait_for(emit_started.wait(), timeout=1.0) + # The emit is blocked -- recovery must already have been scheduled by this point. + assert pool._reconcile_task is not None, "reconcile was not scheduled ahead of the emit" + + emit_release.set() + await asyncio.wait_for(mark_task, timeout=1.0) + await asyncio.sleep(0.05) + await pool.close() + + asyncio.run(scenario()) + + def test_world_unavailable_twice_gives_up_after_the_one_retry() -> None: async def scenario() -> None: outbound = FakeOutbound() @@ -1067,6 +1716,14 @@ async def flaky_lease(*, exclude=frozenset(), abandon=None): assert receipt.failure is not None and receipt.failure.code == "world_unavailable" assert receipt.scenario_attempt == 1 assert receipt.world_index == 0 + # Defensive, not discriminating on its own: this site's own retry-lease `None` return + # already short-circuits BEFORE the emit both before and after the `scenario_retried` + # placement change (the mutation run confirmed it), so this only pins that the property + # continues to hold -- the actual discriminating test is + # `test_cancel_during_the_retry_leases_own_health_probe...` below, the one whose + # mutation-reverted shape this assertion actually catches. + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_retried") == 0 await pool.close() asyncio.run(scenario()) @@ -1084,7 +1741,7 @@ class Provisioner(FakeProvisioner): async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: if runtime.world_index == 1: cancel_flag["v"] = True - async with self._serialized(): + async with self._serialized(f"healthy(w{runtime.world_index})"): return runtime.state is RuntimeState.READY outbound = FakeOutbound() @@ -1106,6 +1763,11 @@ def check(world: Any, calls: Any) -> None: assert receipt.failure is not None and receipt.failure.code == "world_unavailable" assert receipt.scenario_attempt == 1 assert receipt.world_index == 0 + # The retry lease itself succeeded (world 1 granted), but attempt 2's own cancel + # re-check bailed before it ever proceeded -- `scenario_retried` sits right next to + # attempt 2's `scenario_started` now, so neither fires here either. + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_retried") == 0 await pool.close() asyncio.run(scenario()) @@ -1334,6 +1996,89 @@ async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> An asyncio.run(scenario()) +def test_driver_crashed_reports_the_call_summary_when_the_call_step_already_ran() -> None: + # A crash AFTER a successful call step -- here, `world.read_only()` blowing up while building + # the check-phase handle -- must not report `call: null`. The call demonstrably ran; + # outbound-channels.md Channel 2's errored-receipt body only allows `null` when the call + # never started. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenSecondReadOnly: + def __init__(self) -> None: + self.calls = 0 + + def read_only(self) -> Any: + self.calls += 1 + if self.calls == 1: + return object() # used by the default no-op ready_fn; never touched + raise RuntimeError("check-phase read_only() blew up") + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return BrokenSecondReadOnly() + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.call is not None + assert receipt.call.duration_ms == 5000 # _call_outcome()'s fixture value -- the call ran + await pool.close() + + asyncio.run(scenario()) + + +def test_a_retrys_driver_crashed_receipt_does_not_carry_the_previous_attempts_call_summary() -> None: + # `_ScenarioContext` is shared across both attempts of a retry (`_run_scenario` recurses + # with the same `context` object) -- `world_index`/`attempt` were refreshed on entry but + # `call` was not, so an attempt-1 outcome that reached the call step + # (e.g. `evidence_missing`, retried onto a fresh world) left attempt 1's `CallSummary` sitting + # on the context for attempt 2 to inherit if attempt 2 crashed before making its own call. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class WorldForAttempt: + def __init__(self, world_index: int) -> None: + self.world_index = world_index + self.rng = None + + def read_only(self) -> Any: + if self.world_index == 1: + # attempt 2's own world -- crashes before its call step ever runs. + raise RuntimeError("attempt 2's read_only() blew up before its own call step") + return object() # attempt 1's world -- used by the default no-op ready_fn + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return WorldForAttempt(runtime.world_index) + + # Attempt 1's call genuinely ran (started_at/duration_ms distinct from any fixture default) + # but captured zero tool calls -- `evidence_missing`, retried onto a fresh world. + runner = FakeCallRunner( + {"s1": hs.CallOutcome(calls=(), turns=1, started_at="A1", ended_at="A1-end", duration_ms=1111)} + ) + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.scenario_attempt == 2 + assert receipt.world_index == 1 + assert receipt.call is None # attempt 2 never reached its own call step + await pool.close() + + asyncio.run(scenario()) + + def test_exactly_one_receipt_per_scenario_key_even_when_one_scenario_crashes() -> None: # T7/B3: `gather(return_exceptions=True)` + the try/finally around the leased region must # never produce zero or duplicate receipts for any scenario. @@ -1379,6 +2124,136 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_a_fence_stops_the_run_from_launching_further_scenarios() -> None: + # `HostedFencedError`/`HostedChannelFailedError` used to be swallowed as best-effort telemetry + # by `_emit`/`_log`/`mark_unhealthy`'s emit, so a superseded attempt ran its ENTIRE scenario + # set and billed a whole attempt's worth of simulated calls after the platform had already + # fenced it (outbound-channels.md: 401/403 -> "stop emitting, exit code 3 ... never an infra + # retry"). A fence on the first scenario's own + # `scenario_started` emit must stop every later scenario from ever launching. + async def scenario() -> None: + class FencingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + if scenario_key == "s0": + raise HostedFencedError( + ChannelError(ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + await super().scenario_started( + scenario_key=scenario_key, world_index=world_index, scenario_attempt=scenario_attempt + ) + + outbound = FencingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + n = 5 + runner = FakeCallRunner({f"s{i}": _call_outcome(calls=(hs.Call(name="x", arguments={}),)) for i in range(n)}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(n)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert isinstance(result.fenced, HostedFencedError) + # s0's fence lands before the call step; if the run had kept launching, s1-s4 would have + # reached the call step too (their own `scenario_started` succeeds). + assert runner.calls == [] + assert outbound.receipts == [] # a superseded attempt emits no receipts + await pool.close() + + asyncio.run(scenario()) + + +def test_a_fence_landing_before_the_world_is_resolved_releases_it_instead_of_demoting_it() -> None: + # A `_FATAL_OUTBOUND` escaping through the first `scenario_started` emit reached + # `_run_scenario`'s `finally` with `world_resolved` still `False`, and `mark_unhealthy()` + # there demoted a world that never did anything wrong -- a + # false `world_unhealthy` emitted after the run had already stopped emitting, and a + # `_schedule_reconcile()` call that spent a `provision()` attempt on a job that already stopped. + async def scenario() -> None: + class FencingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + raise HostedFencedError( + ChannelError(ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + + outbound = FencingOutbound() + pool, provisioner = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s0": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s0", "id-0", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert [event for event, _ in outbound.events if event == "world_unhealthy"] == [] + # If a reconcile were (wrongly) scheduled, close() waits for it to finish before + # returning -- checking `provision_calls` only after close() makes this deterministic. + await pool.close() + assert provisioner.provision_calls == 1 # only start()'s own call + + asyncio.run(scenario()) + + +def test_a_channel_failed_error_latches_the_same_fenced_path() -> None: + # The same latch, reached through `WorldPool.mark_unhealthy`'s own emit rather than the + # scheduler's `_emit` -- `HostedChannelFailedError` (404x3, "finalize platform_sync") is just + # as fatal as a fence and must stop the run the same way. + async def scenario() -> None: + class ChannelFailingOutbound(FakeOutbound): + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + raise HostedChannelFailedError( + ChannelError(ChannelOutcome.CHANNEL_FAILED, None, "not_found", "channel gone") + ) + + outbound = ChannelFailingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="boom") + assert pool.fenced is not None + assert isinstance(pool.fenced, HostedChannelFailedError) + await pool.close() + + asyncio.run(scenario()) + + +def test_an_attempt_superseded_error_latches_the_same_fenced_path() -> None: + # `HostedAttemptSupersededError` (409 `attempt_superseded`) is the third member of + # outbound.py's `ChannelState` "stop emitting" latch, not just the two originally caught -- + # it is "a fence in substance" per that module's own docstring. Without it in + # `_FATAL_OUTBOUND`, a superseded attempt took the best-effort branch instead and drained its + # entire scenario list against a channel that refuses every request. + async def scenario() -> None: + class SupersedingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + if scenario_key == "s0": + raise HostedAttemptSupersededError( + ChannelError(ChannelOutcome.PERMANENT_ITEM, None, "attempt_superseded", "attempt was superseded") + ) + await super().scenario_started( + scenario_key=scenario_key, world_index=world_index, scenario_attempt=scenario_attempt + ) + + outbound = SupersedingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + n = 5 + runner = FakeCallRunner({f"s{i}": _call_outcome(calls=(hs.Call(name="x", arguments={}),)) for i in range(n)}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(n)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert isinstance(result.fenced, HostedAttemptSupersededError) + assert runner.calls == [] + assert outbound.receipts == [] + await pool.close() + + asyncio.run(scenario()) + + def test_cancel_after_the_first_scenario_skips_the_rest() -> None: # T3/TH-2: the original test's `cancel_requested=lambda: True` was true before `run()` was # even called, so nothing ever launched and the "in-flight scenario finishes" behavior was @@ -1412,6 +2287,15 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.C scenario_key="s2", scenario_id="id-2", scenario_attempt=1, world_index=None, status="skipped", sub_goals=(), evaluations=(), call=None, failure=None, ) + # (outbound-channels.md v1.3 Sequencing: "terminal event -> skipped receipts -> + # manifest"): `run()` only SYNTHESIZES the skipped receipts -- emitting them is the + # caller's job, done after its own terminal event. Only the real "s0" receipt should be + # on the wire at this point. + assert [r.scenario_key for r in outbound.receipts] == ["s0"] + await scheduler.emit_skipped_receipts(result) + assert [r.scenario_key for r in outbound.receipts] == ["s0", "s1", "s2"] + assert outbound.receipts[1] == result.receipts[1] + assert outbound.receipts[2] == result.receipts[2] await pool.close() asyncio.run(scenario()) @@ -1464,6 +2348,12 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.C assert statuses["s3"] == "skipped" started = [event for event, _ in outbound.events if event == "scenario_started"] assert len(started) == 1 # s2/s3 were genuinely never launched + # Same ordering guarantee -- only the real "s1" receipt is on the wire until the + # caller explicitly asks for the synthesized `skipped` ones. + assert [r.scenario_key for r in outbound.receipts] == ["s1"] + await scheduler.emit_skipped_receipts(result) + skipped_keys = sorted(r.scenario_key for r in outbound.receipts if r.status == "skipped") + assert skipped_keys == ["s2", "s3"] asyncio.run(scenario()) From bd463cb099d2da27de488686b82a492740a06f5c Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 19:20:50 +0530 Subject: [PATCH 12/20] =?UTF-8?q?feat(harness):=20hosted=20entrypoint=20?= =?UTF-8?q?=E2=80=94=20lifecycle,=20outbound=20wiring,=20exit-code=20contr?= =?UTF-8?q?act,=20W=20from=20job=20parallelism?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_entrypoint.py | 1705 +++++++++++++++++- src/fi/simulate/runtime/spec.py | 3 + tests/harness/test_hosted_entrypoint.py | 2187 +++++++++++++++++++++++ 3 files changed, 3877 insertions(+), 18 deletions(-) create mode 100644 tests/harness/test_hosted_entrypoint.py diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 2a4f792a..03fa054b 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -1,41 +1,1710 @@ -"""Standalone entrypoint run inside one isolated hosted ALK sandbox. +"""The hosted guest's `main()` — `hosted-execution-seams.md` v1.14 §0/§4/§5, `outbound-channels.md` +v1.3, `world-handle-interface.md` v3.4. Everything between "sandbox starts" and "exit code": read +`/work/job.json`, load the platform capability file, run §2e preflight, pre-allocate scenarios +against `endpoints.scenarios`, provision the world pool, drive the scenario loop, adapt its events/ +receipts/artifacts onto the real outbound clients, and honor the exit-code contract (§0.6). -The platform creates the typed job and prepares a source checkout through its repository -integration. It does not execute harness stages. This process consumes those inputs and runs the -same ``HarnessExecutor`` used locally. +Ownership boundary (read this before touching orchestration order): the stages BEFORE bundle +authoring — `understanding_agent`, `generating_environment`, `building_environment`'s bundle-write +half — belong to Rishav's stages (contract §6) and are not implemented anywhere in this repo yet. +This module does not attempt them. `BundleSource`/`ScenarioSource` below are the seams a later +change wires the real stages through; until then their defaults raise a typed, clearly-named error +rather than silently producing a fake bundle or a fake scenario set. + +`process_runtime.py`, `hosted_scheduler.py`, and `outbound.py` were being fixed by parallel workers +while this module was written. It codes against the four frozen contracts and the cross-review +obligation lists, not against those files' exact HEAD. """ from __future__ import annotations import argparse import asyncio +import hashlib +import json +import logging +import random +import signal +import time +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path +from typing import Any, Callable, Protocol, Sequence + +from . import outbound as ob +from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, ProcessKind, load_bundle_v2 +from .hosted_scheduler import ( + CallOutcome, + CallRunner, + HostedScheduler, + ResultReceipt, + RunResult, + Scenario, + World, + WorldFactory, + WorldPool, + WorldProvisioner, +) +from .job import ( + ArtifactLevel, + ExecutionMode, + FailureDomain, + HarnessArtifactPolicy, + HarnessJob, + HarnessStage, +) +from .process_preflight import PreflightError, preflight_bundle +from .process_runtime import ( + EnvironmentRuntime, + ProcessRuntimeError, + ProcessRuntimeProvider, + RuntimeEndpoint, +) +from .world.handle import HostedWorld +from .world.stores.postgres import AttachedPostgresStore + +logger = logging.getLogger(__name__) + +# --- §0.6 exit-code contract -------------------------------------------------------------------- +# +# 0 = any terminal stage reached (completed/failed/canceled), outbox flushed. 3 = fenced/superseded +# (HostedFencedError anywhere -> stop emitting, no terminal event, exit 3). 4 = the terminal was +# decided but the final drain could not deliver it (the events channel failed, or the platform +# permanently rejected the terminal item itself) -- the gateway treats it exactly like a crash +# (infrastructure retry, fresh channels), but the distinct code tells operators the job DID reach a +# terminal state, unlike a genuine crash. Any other non-zero = the guest crashed before a terminal +# state -- the gateway records `infrastructure`. Capabilities-file failures are explicitly carved +# out of the "any other non-zero" bucket only by CODE (they must never be 3, per +# outbound-channels.md v1.3's rejection table); they still use a non-zero exit here since there is +# no channel to report a terminal FAILED event through. +EXIT_OK = 0 +EXIT_FENCED = 3 +EXIT_TERMINAL_UNDELIVERED = 4 # terminal reached but not provably flushed on the final drain. +EXIT_BOOT_FAILURE = 1 # capabilities.json could not be loaded -- no channel, no event (v1.3 table). +EXIT_CRASHED = 2 # an uncaught failure before any terminal stage was reached. + +# Cancellation signal (spine §0 step 7 / outbound-channels.md "Cancellation signal"). The task +# brief that spawned this module named `/work/cancel.json`; the two frozen contracts that actually +# define this file (seams §0 step 7, outbound-channels "Cancellation signal") both name +# `/run/futureagi/cancel.json`. Contracts are authoritative over a task brief. +CANCEL_SIGNAL_PATH = "/run/futureagi/cancel.json" + +# STUCK DECISION (fail-safe/reversible; contract gap): the invocation contract +# (spine §0 step 5) pins the entrypoint's argv to exactly `job --source ... --output ...`; `--output` +# is `/work/artifacts` (spine layout block), so `work_directory` (what `preflight_bundle`/ +# `provision`/`write_build_output` all want -- the `/work` root) is derived as `output.parent` +# rather than taken as a separate flag, since the frozen invocation line has no room for one. +# `bundle_dir` has no convention anywhere in the frozen documents at all (bundle authoring is not +# built yet); `DEFAULT_BUNDLE_DIR_NAME` is this module's own placeholder location, overridable via +# `BundleSource` injection so a later change can point it at wherever the real authoring stage ends +# up writing without touching this file's orchestration. +DEFAULT_BUNDLE_DIR_NAME = "bundle" +EVENTS_SPOOL_DIR_NAME = "outbound-spool" # must not live under work_directory/"artifacts". + +SECRETS_PATH = Path("/run/futureagi/secrets.json") -from .executor import HarnessExecutor -from .job import ExecutionMode, HarnessJob, HarnessStage +# ================================================================================================= +# Boot -- job.json + capabilities.json (§0.2/§0.4; outbound-channels.md Authentication). +# ================================================================================================= -async def _run(job_path: Path, source: Path, output: Path) -> int: + +def load_job(job_path: Path) -> HarnessJob: + """§0.2: `/work/job.json` is the provisioner's job-identity and configuration source.""" job = HarnessJob.model_validate_json(job_path.read_text(encoding="utf-8")) if job.execution is not ExecutionMode.HOSTED: raise ValueError("hosted_entrypoint_requires_hosted_job") - status = await HarnessExecutor().run(job, source=source, output=output) - print(status.model_dump_json(exclude_none=True), flush=True) - return 0 if status.stage is HarnessStage.COMPLETED else 1 + return job -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="alk-harness-worker") - parser.add_argument("job", type=Path, help="typed HarnessJob JSON") - parser.add_argument( - "--source", required=True, type=Path, help="sandbox-owned source checkout" +def resolve_parallelism(job: HarnessJob) -> int: + """`job.runtime.parallelism` = W (glossary). Returns the RAW requested value, never + clamped -- §2e.7 reserves `parallelism_out_of_range` for a W outside 1..8, and + `preflight_bundle` (called BEFORE any provisioning) is the enforcement point for the UPPER + bound. The lower bound never reaches preflight at all: `RuntimeRequirements.parallelism`'s own + `ge=1` rejects a non-positive W earlier, at `load_job`, as a deliberate defense-in-depth floor + (harmless today since the gateway caps W at admission before a job is ever built). Clamping + here would silently launder an in-range-but-wrong W and make `parallelism_out_of_range` + permanently unreachable for the upper bound.""" + return job.runtime.parallelism + + +def job_secret_purposes(job: HarnessJob) -> dict[str, str]: + """§1: `agent.secret_refs` alias -> `SecretRef.purpose`, the shape `preflight_bundle` wants.""" + return {alias: ref.purpose for alias, ref in job.agent.secret_refs.items()} + + +def peek_secret_values(secrets_path: Path) -> tuple[str, ...]: + """A non-destructive read of `/run/futureagi/secrets.json`'s VALUES ONLY, for outbound + redaction (`extra_secret_values` — outbound.py's `redact_outbound_text`). §0.3's lifetime rule + ("the provisioner loads this file into memory at startup and deletes it") is honored by + `ProcessRuntimeProvider` itself; this is an additional, side-effect-free read (no unlink) done + once at boot so free-text event/log/failure fields can be scrubbed of every resolved secret + value, not just URL userinfo. Never fatal: a missing/malformed file just means no extra values + to scrub, matching `redact_outbound_text`'s own `extra_secret_values=()` default.""" + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return () + if not isinstance(raw, dict): + return () + return tuple(str(value) for value in raw.values() if value) + + +# ================================================================================================= +# Bundle source -- §2 bundle authoring is not this module's (or built anywhere yet); injectable. +# ================================================================================================= + + +class BundleUnavailableError(RuntimeError): + """Raised by a `BundleSource` when no bundle could be produced/located. Mapped the same way as + a `PreflightError` (FAILED, `FailureDomain.ENVIRONMENT`, stage `validating_environment`) — + from the entrypoint's point of view "no bundle" and "bad bundle" are the same class of + environment-authoring fault, and §2e's own failure table has no separate code for it.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +class BundleSource(Protocol): + def load( + self, job: HarnessJob, *, source: Path, work_directory: Path + ) -> tuple[EnvironmentBundleV2, Path]: ... + + +# §2e's closed failure-code table (hosted-execution-seams.md) -- `BundleV2Error` has no typed +# `.code` (a bare `RuntimeError`), so `DefaultBundleSource.load` below string-splits +# its message on ":". `bundle_manifest_missing` (one of the four messages `load_bundle_v2` can +# raise) is not in this table -- a real contract gap -- so both that code AND anything else the +# split produces outside this frozen set fall back to `bundle_manifest_invalid` rather than +# shipping an unlisted code across the outbound seam. +_SECTION_2E_CODES = frozenset( + { + "compose_not_hosted", "engine_unsupported", "no_sql_store", "seed_missing", + "seed_strategy_unsupported", "sentinel_shape_mismatch", "store_protocol_unsupported", + "capability_engine_mismatch", "store_service_not_managed", "reserved_name", + "unknown_placeholder", "unknown_field", "secret_in_bundle", "secret_unclaimed", + "secret_missing", "build_requires_root", "user_assignment_invalid", + "configuration_name_duplicate", "configuration_name_required", + "configuration_name_reserved", "sentinel_shape_invalid", "capability_unresolved", + "service_unresolved", "control_service_unresolved", "process_name_duplicate", + "inputs_digest_mismatch", "bundle_schema_unsupported", "bundle_manifest_invalid", + "bundle_manifest_drifted", "bundle_digest_mismatch", "bundle_digest_invalid", + "inputs_digest_invalid", "file_sha256_invalid", "source_digest_invalid", + "bundle_file_missing", "bundle_file_changed", "bundle_file_unlisted", + "bundle_symlink_forbidden", "bundle_path_unsafe", "depends_on_unresolved", + "depends_on_cycle", "seed_file_missing", "seed_file_unlisted", "process_count_exceeded", + "parallelism_out_of_range", "evidence_seam_required", "processes_required", + "processes_and_seed_forbidden", "document_only_for_compose", + "compose_runtime_requires_document", "build_command_step_empty", + "started_check_requires_exactly_one_of_port_or_log_marker", "resolved_secret_forbidden", + "capability_slug_invalid", "process_name_invalid", "fixed_port_reserved", + } +) + + +def _bundle_unavailable_code(raw_message: str) -> str: + code = raw_message.split(":", 1)[0].strip() + return code if code in _SECTION_2E_CODES else "bundle_manifest_invalid" + + +class DefaultBundleSource: + """Looks for an already-authored bundle at `work_directory / bundle_dir_name`. This is a + placeholder location this module invented (see the module-level STUCK DECISION note) — a real + bundle-authoring stage should either write there or be wired in via its own `BundleSource`.""" + + def __init__(self, bundle_dir_name: str = DEFAULT_BUNDLE_DIR_NAME) -> None: + self._bundle_dir_name = bundle_dir_name + + def load( + self, job: HarnessJob, *, source: Path, work_directory: Path + ) -> tuple[EnvironmentBundleV2, Path]: + del job, source # unused by the default (a real stage would author from these) + bundle_dir = work_directory / self._bundle_dir_name + try: + manifest = load_bundle_v2(bundle_dir) + except BundleV2Error as exc: + raise BundleUnavailableError(_bundle_unavailable_code(exc.args[0]), str(exc)) from exc + return manifest, bundle_dir + + +# ================================================================================================= +# Scenario source -- generation is Karthik's contract (in review, not available here); the +# pre-allocation CALL is this module's (ScenariosClient below). Injectable for the same reason as +# BundleSource: the glue between "generated scenarios" and "pre-allocated against the platform" can +# only be finished once that contract's payload shape lands. +# ================================================================================================= + + +class ScenarioSourceNotWired(RuntimeError): + """The default `ScenarioSource` — no Scenario Generation Contract implementation exists in this + repo yet. Raised rather than fabricating scenarios, and mapped to FAILED / `platform_sync` / + `validating_scenarios`, matching spine §5 step 3.5's own failure mapping for a pre-allocation + that never completes.""" + + +class ScenarioSource(Protocol): + async def build( + self, + job: HarnessJob, + bundle: EnvironmentBundleV2, + scenarios_client: "ScenariosClient", + *, + pool: WorldPool, + world_factory: WorldFactory, + ) -> Sequence[Scenario]: ... + + +class NotWiredScenarioSource: + async def build( + self, + job: HarnessJob, + bundle: EnvironmentBundleV2, + scenarios_client: "ScenariosClient", + *, + pool: WorldPool, + world_factory: WorldFactory, + ) -> Sequence[Scenario]: + del job, bundle, scenarios_client, pool, world_factory + raise ScenarioSourceNotWired( + "no ScenarioSource wired -- scenario generation is not implemented in this repo yet " + "(Scenario Generation Contract, in review)" + ) + + +# ================================================================================================= +# §4.5b -- a single provider mutex serializing every provision/reset/close/healthy call. Explicitly +# this module's duty per the obligations list. `WorldPool` (hosted_scheduler.py) already serializes +# provision/reset/close through its own `_provider_lock`, but NOT `healthy()` (by design — its own +# docstring reads v1.11 §4.5b's non-reentrancy sentence as naming only provision/reset/close); this +# wrapper is the belt-and-suspenders version that holds for all four regardless of what the +# scheduler's own lock covers today, and is safe to layer under it (two distinct `asyncio.Lock` +# objects on a single-threaded event loop cannot deadlock each other). +# ================================================================================================= + + +class SerializingProvider: + def __init__(self, provider: WorldProvisioner) -> None: + self._provider = provider + self._lock = asyncio.Lock() + + @property + def name(self) -> str: + # §4's `RuntimeProvider` Protocol declares `name: str` ("retained for logging only"); + # this wrapper otherwise hides it, so any future `provider.name` read would AttributeError. + return getattr(self._provider, "name", "") + + async def provision( + self, + bundle: Any, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, + contract: Any | None = None, + instances: int = 1, + ) -> list[EnvironmentRuntime]: + async with self._lock: + return await self._provider.provision( + bundle, + source=source, + bundle_dir=bundle_dir, + work_directory=work_directory, + contract=contract, + instances=instances, + ) + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._lock: + await self._provider.reset(runtime, work_directory=work_directory) + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._lock: + return await self._provider.healthy(runtime, work_directory=work_directory) + + async def close(self, *, work_directory: Path) -> None: + async with self._lock: + await self._provider.close(work_directory=work_directory) + + +# ================================================================================================= +# WorldFactory -- real HostedWorld instances, fed by build.json's row counts (never +# a partial map). +# ================================================================================================= + + +class WorldFactoryError(RuntimeError): + """The provisioner handed back a runtime this factory cannot build a `World` for — a bug + upstream (no postgres endpoint despite §2e's `no_sql_store` guarantee, or `build.json` missing + the row counts for that store), never a scenario-code fault.""" + + +# §2f's closed build/run failure-code table (hosted-execution-seams.md §4.6) -> FailureDomain. +# LOCAL to this module for now -- no module owns this map today. `spawn_failed` is the one code the table itself +# splits by process kind ("infrastructure if a managed engine, agent if source"), so it is resolved +# by a manifest lookup in `_process_runtime_error_domain` below rather than a flat entry here. +_SECTION_2F_DOMAIN: dict[str, FailureDomain] = { + "source_tree_unavailable": FailureDomain.ENVIRONMENT, + "build_failed": FailureDomain.AGENT, + "runtime_unsupported": FailureDomain.ENVIRONMENT, + "depends_on_timeout": FailureDomain.INFRASTRUCTURE, + "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, + "seed_failed": FailureDomain.ENVIRONMENT, + "store_statement_failed": FailureDomain.INFRASTRUCTURE, +} + + +def _process_runtime_error_domain( + exc: ProcessRuntimeError, manifest: EnvironmentBundleV2 +) -> FailureDomain: + if exc.code == "spawn_failed": + for process in manifest.processes: + if process.name == exc.process: + return ( + FailureDomain.INFRASTRUCTURE + if process.kind is ProcessKind.MANAGED + else FailureDomain.AGENT + ) + return FailureDomain.INFRASTRUCTURE # unresolvable process name -- the honest default + return _SECTION_2F_DOMAIN.get(exc.code, FailureDomain.INFRASTRUCTURE) # internal_* etc. + + +_SECTION_2F_CODES: frozenset[str] = frozenset(_SECTION_2F_DOMAIN) | {"spawn_failed"} + + +def _section_2f_code(code: str) -> str: + # §2f is closed (contract §4.6) -- `process_runtime.py`'s own `internal_*` codes, and this + # module's untyped-exception fallback, must never cross the outbound seam unlabeled, matching + # the discipline `_bundle_unavailable_code` already applies to §2e. The real code is + # still visible on the wire -- it stays in `message` (`ProcessRuntimeError.__str__` embeds it, + # and the untyped-exception call site prefixes it explicitly). + return code if code in _SECTION_2F_CODES else "spawn_failed" + + +def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint: + for endpoint in runtime.endpoints.values(): + if endpoint.protocol == "postgres": + return endpoint + raise WorldFactoryError( + f"world {runtime.world_index}: no postgres-protocol endpoint in {sorted(runtime.endpoints)} " + "-- §2e's no_sql_store rule should make this unreachable" ) - parser.add_argument( - "--output", required=True, type=Path, help="job artifact directory" + + +def load_build_output(work_directory: Path) -> dict[str, Any]: + """`write_build_output` (process_runtime.py) writes `/artifacts/build.json`. + Read fresh each call — cheap, and the row counts are immutable after baseline freeze, so + re-reading is simpler than a cache invalidation story for the same modest cost.""" + path = work_directory / "artifacts" / "build.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise WorldFactoryError(f"build.json unreadable at {path}: {exc}") from exc + + +def row_counts_for_capability(build_output: dict[str, Any], capability: str) -> dict[str, int]: + for store in build_output.get("stores", []): + if store.get("capability") == capability: + counts = store.get("row_counts") or {} + return {str(name): int(count) for name, count in counts.items()} + raise WorldFactoryError( + f"build.json has no store entry for capability {capability!r} — the provisioner " + "guarantees a complete row-count map per store, so this bundle's build output is malformed" ) + + +class ProcessWorldFactory: + """Builds a real `HostedWorld` over the runtime's postgres endpoint. `AttachedPostgresStore` + (not the bare `PostgresStore`) is the correct base here — it takes a raw DSN and never manages + a container's own lifecycle, matching a hosted world where `ProcessRuntimeProvider` already + owns the postgres process.""" + + def __init__(self, work_directory: Path) -> None: + self._work_directory = work_directory + + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> World: + endpoint = _find_postgres_endpoint(runtime) + build_output = await asyncio.to_thread(load_build_output, self._work_directory) + row_counts = row_counts_for_capability(build_output, endpoint.capability) + store = AttachedPostgresStore(endpoint.address) + return await asyncio.to_thread( + HostedWorld, store, runtime.world_index, rng, row_counts + ) + + +# ================================================================================================= +# CallRunner -- the real voice track wires this later. Typed NotWired default only. +# ================================================================================================= + + +class CallRunnerNotWired(RuntimeError): + """Raised by `NotWiredCallRunner`. `hosted_scheduler._execute` treats any exception out of + `CallRunner.run` (other than `WorldUnavailable`/`CallAborted`) as `call_failed` + (`FailureDomain.INFRASTRUCTURE`, retried once) — so a job run with nothing wired here degrades + every scenario to one retry-then-errored receipt rather than crashing the process.""" + + +class NotWiredCallRunner: + async def run(self, scenario: Scenario, runtime: EnvironmentRuntime) -> CallOutcome: + del scenario, runtime + raise CallRunnerNotWired( + "no CallRunner wired -- the live voice-simulation call runner is a separate track" + ) + + +# ================================================================================================= +# Scenario pre-allocation -- a thin client against endpoints.scenarios (outbound-channels.md v1.3 +# Authentication: bearer + X-Harness-Fence, `{"result": {...}}` envelope, job-scoped idempotent). +# Previously unowned; owned by this module now. +# ================================================================================================= + + +class ScenarioPreallocationError(RuntimeError): + def __init__(self, error: ob.ChannelError | None) -> None: + self.error = error + super().__init__("scenario pre-allocation failed" if error is None else error.message) + + +class ScenariosClient: + """CROSS-DOC GAP: the Scenario Generation Contract that defines the exact + `provision`/`begin` payload and path shape is Karthik's, "in review," and not available to this + module. `provision_path`/`begin_path` are constructor-injectable placeholders rather than a + guess baked into the URL, so the real paths can be supplied without touching this class once + that contract lands. Shares `channel_state` with the other three channels (a fence on any one + must stop all of them, per outbound.py's own `ChannelState` docstring).""" + + def __init__( + self, + capabilities: ob.HostedCapabilities, + transport: ob.Transport | None = None, + *, + retry_policy: ob.RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + channel_state: ob.ChannelState | None = None, + provision_path: str = "provision/", + begin_path: str = "begin/", + ) -> None: + self._capabilities = capabilities + self._transport = transport or ob.RequestsTransport() + self._retry_policy = retry_policy or ob.RetryPolicy() + self._sleep = sleep + self._rng = rng + self._channel_state = channel_state or ob.ChannelState() + self._provision_path = provision_path + self._begin_path = begin_path + + def provision(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + return self._post(self._provision_path, payload, deadline=deadline) + + def begin(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + return self._post(self._begin_path, payload, deadline=deadline) + + def _post( + self, path_suffix: str, payload: dict[str, Any], *, deadline: float | None + ) -> dict[str, Any]: + self._channel_state.check() + url = f"{self._capabilities.endpoints.scenarios}{path_suffix}" + + def perform(_attempt: int) -> ob.TransportResponse: + return self._transport.request( + "POST", url, headers=self._capabilities.auth_headers(), json_body=payload + ) + + try: + response, error = ob._perform_with_retry( + perform, + retry_policy=self._retry_policy, + sleep=self._sleep, + rng=self._rng, + deadline=deadline, + ) + except (ob.HostedFencedError, ob.HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None or response is None: + raise ScenarioPreallocationError(error) + body = response.body if isinstance(response.body, dict) else {} + result = body.get("result") + if not isinstance(result, dict): + raise ScenarioPreallocationError( + ob.ChannelError( + ob.ChannelOutcome.PERMANENT_ITEM, + FailureDomain.PLATFORM_SYNC, + "scenarios_envelope_invalid", + "response body has no {'result': {...}} envelope", + ) + ) + return result + + +# ================================================================================================= +# OutboundPort adapter -- the real emit pipeline: redact -> capabilities.event_builder() -> +# spool.append -> EventsClient.flush(). Also: baseline_frozen/parallelism_degraded from build.json, +# terminal events (exactly one, last), artifact-before-receipt ordering, +# and RunResult.aborted -> TerminalFailure(infrastructure, running, "world_pool_exhausted"). +# ================================================================================================= + + +_TERMINAL_FAILURE_MESSAGE_MAX_CHARS = 4096 # an unbounded `failure.message` can blow +# EVENT_PAYLOAD_MAX_BYTES and hard-reject the WHOLE terminal event; log is the only event type +# that self-truncates. 4KB is ample for a diagnostic message. + + +def _cap_failure_message(message: str) -> str: + if len(message) <= _TERMINAL_FAILURE_MESSAGE_MAX_CHARS: + return message + marker = "…[truncated]" + return message[: _TERMINAL_FAILURE_MESSAGE_MAX_CHARS - len(marker)] + marker + + +# guest-side mirror of outbound-channels.md's artifact level table (Channel 3) -- no module +# owns this table yet (the sealer's own version lives at `artifacts.py::seal_artifacts`, scoped to +# the local-SDK path); this hosted upload path needs its own "guest enforces it first" half. +_ARTIFACT_LEVEL_FORBIDDEN_KINDS: dict[ArtifactLevel, frozenset[ob.ArtifactKind]] = { + ArtifactLevel.METADATA_ONLY: frozenset( + { + ob.ArtifactKind.RECORDING_COMBINED, ob.ArtifactKind.RECORDING_STEREO, + ob.ArtifactKind.RECORDING_CUSTOMER, ob.ArtifactKind.RECORDING_ASSISTANT, + ob.ArtifactKind.TRACE, ob.ArtifactKind.TOOL_TRACE, ob.ArtifactKind.TRANSCRIPT, + ob.ArtifactKind.OTHER, + } + ), + ArtifactLevel.TRACES: frozenset( + { + ob.ArtifactKind.RECORDING_COMBINED, ob.ArtifactKind.RECORDING_STEREO, + ob.ArtifactKind.RECORDING_CUSTOMER, ob.ArtifactKind.RECORDING_ASSISTANT, + ob.ArtifactKind.OTHER, + } + ), + ArtifactLevel.TRACES_AND_RECORDINGS: frozenset({ob.ArtifactKind.OTHER}), + ArtifactLevel.FULL: frozenset(), + # `local-only` is rejected at hosted admission (`local_only_not_hosted`) per the contract -- + # this adapter should never see it for a hosted job; forbid everything as a defensive default. + ArtifactLevel.LOCAL_ONLY: frozenset(ob.ArtifactKind), +} + + +class OutboundAdapter: + """Implements `hosted_scheduler.OutboundPort` plus the extra surface the entrypoint itself + needs (`stage_changed`, `baseline_frozen`, `parallelism_degraded`, `upload_artifact`, + `push_manifest`, `emit_terminal`) — hosted_scheduler.py only names the five methods scenario + code needs; everything else here is this module's own. + + Fencing (HostedFencedError) is caught INTERNALLY by every method, never re-raised: letting it + escape into `HostedScheduler._emit()` (which catches bare `Exception` and tries to log through + the very port that just raised) would silently swallow the fence and let the scheduler keep + working an attempt that can no longer report anything. `is_fenced` is the flag the entrypoint's + orchestration (and `cancel_requested`) polls instead. + """ + + def __init__( + self, + capabilities: ob.HostedCapabilities, + *, + events_spool: ob.OutboundSpool, + events_client: ob.EventsClient, + results_client: ob.ResultsClient, + artifacts_client: ob.ArtifactsClient, + channel_state: ob.ChannelState, + extra_secret_values: tuple[str, ...] = (), + clock: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + flush_window_seconds: float = ob.FLUSH_WINDOW_SECONDS, + ) -> None: + self._capabilities = capabilities + self._spool = events_spool + self._events = events_client + self._results = results_client + self._artifacts = artifacts_client + self._channel_state = channel_state + self._extra_secret_values = extra_secret_values + self._clock = clock + # `event_builder`'s own `extra_secret_values` binding is what lets + # `build_event_record` redact `log.message`/`world_unhealthy.cause`/ + # `baseline_frozen.baseline_ref`/`terminal.failure.{code,message}` for every event this + # adapter emits -- binding it here, alongside identity, gives Channel 1 full redaction coverage. + self._event_builder = capabilities.event_builder(extra_secret_values=extra_secret_values) + self._stage_started = False + self._current_stage = HarnessStage.QUEUED + self._uploaded_digests: set[str] = set() + self._manifest_entries: list[dict[str, Any]] = [] + self._terminal_emitted = False + # §0.6 v1.14 (exit code 4): the terminal record's own spool sequence, and whether the + # platform ever permanently rejected it by name -- `terminal_undelivered` (below) needs to + # tell "this specific record landed" apart from "some flush somewhere failed." + self._terminal_sequence: int | None = None + self._terminal_rejected = False + self._scenario_counts: dict[str, int] = { + "passed": 0, "failed": 0, "errored": 0, "skipped": 0, + } + self._fenced_error: Exception | None = None + self._channel_failed_error: Exception | None = None + # the 120s flush window (§5.5) -- armed once, at whichever comes first: a cancel + # signal (`arm_flush_window` called explicitly by `run_job`'s `cancel_requested`) or the + # terminal event (`emit_terminal` below arms it itself, so no caller can forget). + self._flush_window_seconds = flush_window_seconds + self._flush_window_start: float | None = None + # job.artifacts is only known once job.json is parsed, which happens after this + # adapter is built (capabilities load, and the "no channel on a capabilities failure" + # contract, must come first) -- `configure_artifacts` below is called once it's available; + # this default is never actually exercised in practice, just a safe placeholder shape. + self._artifacts_policy = HarnessArtifactPolicy() + # `recording_headroom_bytes` stays 0 -- this adapter has no visibility into how many + # scenarios are still to run (or how large their recordings will be) at construction time, + # unlike the scheduler; sizing it here would be a guess dressed up as enforcement. + self._budget_tracker = ob.ArtifactBudgetTracker(self._artifacts_policy.max_artifact_bytes) + # `would_admit` (check) and `record` (reserve) must run as one atomic step -- two + # concurrent scenarios at W>1 racing the same remaining budget could otherwise both pass + # the check against a snapshot neither has updated yet. + self._artifact_budget_lock = asyncio.Lock() + + @property + def is_fenced(self) -> bool: + return self._fenced_error is not None + + @property + def terminal_undelivered(self) -> bool: + """The terminal was spooled (`emit_terminal` succeeded) but never confirmed delivered: the + platform permanently rejected the terminal item by name, or the spool's watermark never + reached the terminal's own sequence at all (channel exhaustion, a dead channel, or the + flush window running out before delivery). Exit 0 would claim a flush that provably never + happened. Fencing is checked by the caller first and always wins -- once fenced, whether + the terminal was ALSO undelivered is moot.""" + if self._terminal_sequence is None or self.is_fenced: + return False + return self._terminal_rejected or self._spool.watermark() < self._terminal_sequence + + @property + def scenario_counts(self) -> dict[str, int]: + return dict(self._scenario_counts) + + def configure_artifacts(self, policy: HarnessArtifactPolicy) -> None: + self._artifacts_policy = policy + self._budget_tracker = ob.ArtifactBudgetTracker(policy.max_artifact_bytes) + + def arm_flush_window(self) -> None: + if self._flush_window_start is None: + self._flush_window_start = time.monotonic() + + def deadline(self) -> float | None: + if self._flush_window_start is None: + return None + return self._flush_window_start + self._flush_window_seconds + + def _record_channel_error(self, exc: Exception) -> None: + if isinstance(exc, ob.HostedFencedError): + self._fenced_error = self._fenced_error or exc + else: + self._channel_failed_error = self._channel_failed_error or exc + logger.error("outbound channel latched: %s", exc) + + def _guarded(self, fn: Callable[[], Any]) -> Any: + """Runs one outbound client call. Catches `HostedFencedError`/`HostedChannelFailedError` + so neither escapes as an ordinary exception (see class docstring).""" + try: + self._channel_state.check() + except ( + ob.HostedFencedError, ob.HostedChannelFailedError, ob.HostedAttemptSupersededError, + ) as exc: + self._record_channel_error(exc) + return None + try: + return fn() + except (ob.HostedFencedError, ob.HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + self._record_channel_error(exc) + return None + + # -- events ------------------------------------------------------------------------------- + + def _emit_event( + self, *, stage: HarnessStage, type_: ob.OutboundEventType, payload: dict[str, Any] + ) -> None: + if self.is_fenced: + return # "stop emitting" -- no event of any type once fenced. + if self._terminal_emitted: + # `_bounded_close()` runs after the terminal is spooled -- an in-flight reconcile + # inside it can still call back into world_unhealthy/log. v1.3's "terminal ... exactly + # one, last emitted" is a hard invariant: anything after it is dropped locally, not + # spooled, rather than silently landing after the event the platform already finalized on. + # This also drops the rejected-event error log for anything the platform rejects on the + # SAME flush that carries the terminal -- diagnostic-quality only, since the rejected + # bytes are still recoverable as a `log`-kind artifact. + logger.warning( + "outbound event dropped after terminal: type=%s stage=%s", type_.value, stage.value + ) + return + event_id = f"event_{uuid.uuid4().hex}" + record = self._event_builder( + event_id=event_id, emitted_at=self._clock(), stage=stage, type=type_, payload=payload, + ) + spooled = self._spool.append(record) + if type_ is ob.OutboundEventType.TERMINAL: + self._terminal_sequence = spooled.sequence + self._current_stage = stage + + async def _aemit_event( + self, *, stage: HarnessStage, type_: ob.OutboundEventType, payload: dict[str, Any] + ) -> None: + # the spool append fsyncs the file AND its directory -- routed off the event loop so + # it never stalls every other concurrently-running scenario at W>1. + await asyncio.to_thread(self._emit_event, stage=stage, type_=type_, payload=payload) + + def stage_changed(self, to: HarnessStage) -> None: + frm = self._current_stage.value if self._stage_started else None + self._stage_started = True + self._emit_event( + stage=to, type_=ob.OutboundEventType.STAGE_CHANGED, payload={"from": frm, "to": to.value}, + ) + + def baseline_frozen(self, *, inputs_digest: str, baseline_ref: str) -> None: + self._emit_event( + stage=HarnessStage.VALIDATING_ENVIRONMENT, + type_=ob.OutboundEventType.BASELINE_FROZEN, + payload={"inputs_digest": inputs_digest, "baseline_ref": baseline_ref}, + ) + + def parallelism_degraded(self, *, requested: int, effective: int, reason: str) -> None: + self._emit_event( + stage=HarnessStage.VALIDATING_ENVIRONMENT, + type_=ob.OutboundEventType.PARALLELISM_DEGRADED, + payload={"requested": requested, "effective": effective, "reason": reason}, + ) + + def flush_events(self, *, deadline: float | None = None) -> ob.EventsFlushResult | None: + return self._guarded(lambda: self._events.flush(deadline=deadline)) + + async def aflush_events(self, *, deadline: float | None = None) -> None: + result = await asyncio.to_thread(self.flush_events, deadline=deadline) + # a rejected event's own payload never reaches the platform any other way -- surface + # it via a `log` event (error) and keep the bytes recoverable as a `log`-kind artifact, + # keyed off `dropped_records` (captured before the spool physically drops them). + if result is None or not result.rejected: + return + if self._terminal_sequence is not None and any( + entry.get("sequence") == self._terminal_sequence for entry in result.rejected + ): + # A permanent-item rejection is never retried -- the spool physically drops the record, + # so no later flush can ever redeliver it. + self._terminal_rejected = True + dropped_by_sequence = {record.sequence: record for record in result.dropped_records} + for entry in result.rejected: + sequence = entry.get("sequence") + await self.log( + level="error", + message=( + f"event sequence={sequence} rejected by the platform: " + f"{entry.get('code', 'unknown')}: {entry.get('message', '')}" + ), + ) + record = dropped_by_sequence.get(sequence) + if record is not None: + await self.upload_artifact(record.body, kind=ob.ArtifactKind.LOG) + + # -- OutboundPort (hosted_scheduler.py) ---------------------------------------------------- + + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.SCENARIO_STARTED, + payload={ + "scenario_key": scenario_key, "world_index": world_index, + "scenario_attempt": scenario_attempt, + }, + ) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.SCENARIO_RETRIED, + payload={"scenario_key": scenario_key, "from_world": from_world, "to_world": to_world}, + ) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + # world_unhealthy.cause <=200 (WorldUnhealthyPayload hard-rejects over that, so this must + # truncate BEFORE `_aemit_event`, not rely on the builder's own redaction, which runs + # after this call and could not shrink an already-too-long string back into budget). + redacted = ob.redact_outbound_text(cause, self._extra_secret_values) + if len(redacted) > 200: + redacted = redacted[:200] + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.WORLD_UNHEALTHY, + payload={"world_index": world_index, "cause": redacted}, + ) + + async def log(self, *, level: str, message: str) -> None: + await self._aemit_event( + stage=self._current_stage, type_=ob.OutboundEventType.LOG, + payload={"level": level, "message": message}, + ) + + async def receipt(self, receipt: ResultReceipt) -> None: + if self.is_fenced: + return + # counted only once a push is actually attempted -- counting before this point would + # include receipts that were never pushed (and the counts feed the terminal payload). + self._scenario_counts[receipt.status] = self._scenario_counts.get(receipt.status, 0) + 1 + call: dict[str, Any] | None = None + if receipt.call is not None and receipt.call.started_at is not None: + transcript_artifact = receipt.call.transcript_artifact + if transcript_artifact is not None: + bare = transcript_artifact.split(":", 1)[-1] + if bare not in self._uploaded_digests: + # null it rather than shipping a receipt the platform will 422 + # (`artifact_unknown`) wholesale -- the contract explicitly blesses a null + # `transcript_artifact` "named in a log event." + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} references un-acked transcript " + f"artifact {transcript_artifact}; nulling it" + ), + ) + transcript_artifact = None + recording_artifacts: list[str] = [] + for artifact_id in receipt.call.recording_artifacts: + bare = artifact_id.split(":", 1)[-1] if artifact_id else None + if artifact_id and bare not in self._uploaded_digests: + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} references un-acked recording " + f"artifact {artifact_id}; dropping it" + ), + ) + continue + recording_artifacts.append(artifact_id) + ended_at = receipt.call.ended_at + if ended_at is None: + # outbound.CallSummary.ended_at is a required str -- a call that started but + # never finished (CallAborted's partial) would otherwise fail build_result_receipt's + # validation and silently drop the whole receipt (HostedScheduler._emit's blanket + # except swallows it). + ended_at = receipt.call.started_at + await self.log( + level="warning", + message=( + f"receipt for {receipt.scenario_key} has no call.ended_at; substituting " + "started_at" + ), + ) + call = { + "started_at": receipt.call.started_at, + "ended_at": ended_at, + "duration_ms": receipt.call.duration_ms, + "turns": receipt.call.turns, + "transcript_artifact": transcript_artifact, + "recording_artifacts": recording_artifacts, + } + elif receipt.call is not None: + # `hosted_scheduler.CallSummary.started_at` is `str | None`, but + # `outbound.CallSummary.started_at` requires a real timestamp -- per the contract a + # call summary is only present once the call has genuinely started, so a call that + # never started is omitted here rather than shipped with a value that would fail + # `build_result_receipt`'s own validation. + await self.log( + level="warning", + message=f"receipt for {receipt.scenario_key} has no call.started_at; omitting call", + ) + failure: dict[str, Any] | None = None + if receipt.failure is not None: + # Redact before capping -- truncating first can cut a secret in half at + # the boundary and leave exact-substring redaction unable to find the surviving piece. + redacted_failure_message = ob.redact_outbound_text( + receipt.failure.message, self._extra_secret_values + ) + failure = { + "domain": receipt.failure.domain, + "stage": receipt.failure.stage, + "code": receipt.failure.code, + "message": _cap_failure_message(redacted_failure_message), + } + wire = ob.build_result_receipt( + job_id=self._capabilities.job_id, + attempt_id=self._capabilities.attempt_id, + attempt_number=self._capabilities.attempt_number, + scenario_key=receipt.scenario_key, + scenario_id=receipt.scenario_id, + scenario_attempt=receipt.scenario_attempt, + world_index=receipt.world_index, + status=receipt.status, + sub_goals=[ + {"name": g.name, "held": g.held, "reason": g.reason, "judged": g.judged} + for g in receipt.sub_goals + ], + evaluations=[_evaluation_wire(e) for e in receipt.evaluations], + call=call, + failure=failure, + extra_secret_values=self._extra_secret_values, + ) + push_result = await asyncio.to_thread(self._guarded, lambda: self._results.push(wire)) + if push_result is not None and push_result.error is not None: + # The contract's own obligation for a permanent rejection (e.g. 409 receipt_conflict, + # 422 artifact_unknown): "the platform keeps the first; guest logs, no retry." `push()` + # returns this rather than raising, so nothing inspected it before now. + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} rejected by the platform: " + f"{push_result.error.code}: {push_result.error.message}" + ), + ) + await self.aflush_events() + + # -- artifacts (uploaded+acked BEFORE the referencing receipt) -------------------------- + + async def upload_artifact( + self, + data: bytes, + *, + kind: ob.ArtifactKind, + scenario_key: str | None = None, + deadline: float | None = None, + ) -> str | None: + """Returns the `sha256:<64-hex>` id form the wire uses (`CallSummary.transcript_artifact`, + `ArtifactManifestEntry.artifact_id`) — never the bare hex `ArtifactsClient.upload` itself + takes, which is a different, easy-to-mix-up shape (this module's own report notes it).""" + digest = hashlib.sha256(data).hexdigest() + if digest in self._uploaded_digests: + return f"sha256:{digest}" + if self.is_fenced: + return None + # guest-side level admission + budget, both BEFORE the transport is ever touched + # ("the guest enforces it first"). + forbidden = _ARTIFACT_LEVEL_FORBIDDEN_KINDS.get(self._artifacts_policy.level, frozenset()) + if kind in forbidden: + await self.log( + level="error", + message=( + f"artifact upload refused: kind={kind.value} forbidden at " + f"level={self._artifacts_policy.level.value}" + ), + ) + return None + # check-and-reserve atomically, before the actual (slow, concurrency-safe) upload -- + # see the lock's own comment in __init__. A failed upload below leaves the reservation in + # place rather than releasing it (`ArtifactBudgetTracker` has no release primitive): a + # stuck-conservative budget is safe, an under-counted one that lets two racing uploads both + # pass admission is not. + async with self._artifact_budget_lock: + if not self._budget_tracker.would_admit(kind, len(data), digest=digest): + await self.log( + level="error", + message=( + f"artifact upload refused: budget exhausted (kind={kind.value}, " + f"size={len(data)})" + ), + ) + return None + self._budget_tracker.record(kind, len(data), digest=digest) + result = await asyncio.to_thread( + self._guarded, + lambda: self._artifacts.upload( + digest, data, kind=kind, scenario_key=scenario_key, deadline=deadline + ), + ) + if result is None or result.error is not None: + code = result.error.code if result is not None and result.error is not None else "fenced" + await self.log(level="error", message=f"artifact upload failed ({kind.value}): {code}") + return None + self._uploaded_digests.add(digest) + self._manifest_entries.append( + { + "artifact_id": f"sha256:{digest}", "kind": kind.value, "size": len(data), + "scenario_key": scenario_key, + } + ) + return f"sha256:{digest}" + + async def push_manifest(self, *, complete: bool, deadline: float | None = None) -> None: + if self.is_fenced: + return + wire = ob.build_artifact_manifest( + job_id=self._capabilities.job_id, + attempt_id=self._capabilities.attempt_id, + attempt_number=self._capabilities.attempt_number, + entries=list(self._manifest_entries), + complete=complete, + ) + await asyncio.to_thread( + self._guarded, lambda: self._artifacts.push_manifest(wire, deadline=deadline) + ) + + # -- terminal (exactly one terminal event, last emitted) -------------------------------- + + async def emit_terminal( + self, + *, + stage: HarnessStage, + reason: ob.TerminalReason | None = None, + failure: dict[str, Any] | None = None, + ) -> bool: + """Returns whether a terminal event was actually emitted (False when already emitted, or + fenced). No caller reads this return value any more -- `drain()`'s own manifest push is + gated on `is_fenced` instead; kept `bool` since a future caller may still want it.""" + if self._terminal_emitted or self.is_fenced: + return False + if failure is not None and isinstance(failure.get("message"), str): + # redact BEFORE truncating -- the inverse order can cut a secret in half at the 4KB + # boundary, and exact-substring redaction can no longer find the surviving fragment. + redacted = ob.redact_outbound_text(failure["message"], self._extra_secret_values) + failure = {**failure, "message": _cap_failure_message(redacted)} + # the latch is set AFTER a successful append (below), not before -- a raise inside + # `_emit_event` (an oversized payload, an invalid `failure.domain`) must not permanently + # disable the terminal event. + self._emit_event( + stage=stage, + type_=ob.OutboundEventType.TERMINAL, + payload={ + "stage": stage.value, + "reason": reason.value if reason is not None else None, + "failure": failure, + "scenario_counts": dict(self._scenario_counts), + }, + ) + self._terminal_emitted = True + # arm the flush window HERE, unconditionally -- "120s from the cancel signal / TTL / + # terminal event," not only when a cancel was separately observed. + self.arm_flush_window() + return True + + async def flush_terminal(self, *, deadline: float | None = None) -> bool: + """`emit_terminal` only appends the terminal record to the LOCAL spool -- a + caller that pushes something else on the wire right after (skipped receipts) would + otherwise risk `receipt()`'s own trailing `aflush_events()` delivering the terminal as a + side effect of pushing THAT receipt, landing the terminal after it on the wire. Same + bounded loop as `drain()` (a backlog bigger than one `EVENTS_MAX_BATCH` must not + strand the terminal), stopping short of the manifest push -- that still belongs after + skipped receipts, not here. Returns whether a fence was observed.""" + while True: + before = self._spool.watermark() + await self.aflush_events(deadline=deadline) + if self.is_fenced or not self._spool.pending_since_watermark(): + break + if self._spool.watermark() == before or ( + deadline is not None and time.monotonic() >= deadline + ): + break + return self.is_fenced + + async def drain(self, *, complete: bool, deadline: float | None = None) -> bool: + """Best-effort final delivery: events, then the artifact manifest (Sequencing: "terminal + event -> receipts (incl. synthesized skipped) -> manifest" -- receipts are already pushed + individually by `receipt()` as each scenario finishes). Returns whether a fence was + observed during (or before) this call -- the fence most often lands on the very flush + that carries the terminal event, so the caller's exit code must come from THIS return + value, never a fence check taken before drain() ran. + + `aflush_events` delivers at most ONE `EVENTS_MAX_BATCH`-sized batch per call -- a + backlog bigger than that (the rejected-event logging in `aflush_events` can grow one) would otherwise + strand the terminal event, the highest sequence, undelivered while still exiting 0. Loops + until the spool is actually empty, a fence is observed, a flush makes no further progress, + or the deadline is gone -- whichever comes first. + """ + while True: + before = self._spool.watermark() + await self.aflush_events(deadline=deadline) + if self.is_fenced or not self._spool.pending_since_watermark(): + break + if self._spool.watermark() == before or ( + deadline is not None and time.monotonic() >= deadline + ): + break + await self.push_manifest(complete=complete, deadline=deadline) + return self.is_fenced + + +def _evaluation_wire(evaluation: Any) -> dict[str, Any]: + if evaluation.kind == "metric": + return { + # `MetricEvaluation.score: float` coerces `1` -> `1.0`; `build_result_receipt` + # digests the RAW dict before that coercion, so an int here would digest-mismatch + # against the model's own re-derivation and silently drop the receipt. + "name": evaluation.name, "kind": "metric", + "score": float(evaluation.score) if evaluation.score is not None else None, + "reason": evaluation.reason, + } + return { + "name": evaluation.name, "kind": "checkpoint", "passed": evaluation.passed, + "reason": evaluation.reason, + } + + +# ================================================================================================= +# Cancellation -- spine §0 step 7 / outbound-channels.md "Cancellation signal": the gateway writes +# `cancel_path` then sends SIGTERM; the guest stops LAUNCHING new scenarios (not killing what's +# already running) and starts the 120s flush window. +# ================================================================================================= + + +class CancelState: + def __init__(self, path: Path) -> None: + self._path = path + self._sigterm_seen = False + + def note_sigterm(self) -> None: + self._sigterm_seen = True + + def requested(self) -> bool: + return self._sigterm_seen or self._path.exists() + + def reason(self) -> ob.TerminalReason | None: + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + value = raw.get("reason") if isinstance(raw, dict) else None + try: + return ob.TerminalReason(value) + except ValueError: + return None + + +def install_sigterm_handler(cancel_state: CancelState) -> Callable[[], None]: + """Best-effort: `signal.signal` only works on the process's main thread and raises + `ValueError` anywhere else (e.g. inside a test running on a worker thread) — caught and + turned into a no-op restore, since `cancel_requested` still works off the file alone.""" + + def _handler(signum: int, frame: Any) -> None: + del signum, frame + cancel_state.note_sigterm() + + try: + previous = signal.signal(signal.SIGTERM, _handler) + except (ValueError, OSError): + return lambda: None + + def _restore() -> None: + try: + signal.signal(signal.SIGTERM, previous) + except (ValueError, OSError): + pass + + return _restore + + +def default_install_sigterm_handler(cancel_state: CancelState) -> Callable[[], None]: + return install_sigterm_handler(cancel_state) + + +# ================================================================================================= +# Dependency injection -- every seam a test needs to replace with a fake, gathered in one place so +# `run_job` itself stays pure orchestration. +# ================================================================================================= + + +@dataclass +class HostedEntrypointDeps: + load_capabilities: Callable[[], ob.HostedCapabilities] = field( + default=lambda: ob.load_capabilities() + ) + bundle_source: BundleSource = field(default_factory=DefaultBundleSource) + scenario_source: ScenarioSource = field(default_factory=NotWiredScenarioSource) + build_transport: Callable[[], ob.Transport] = field(default=lambda: ob.RequestsTransport()) + build_provider: Callable[[], WorldProvisioner] = field( + default=lambda: ProcessRuntimeProvider() + ) + # The real call runner needs `OutboundAdapter.upload_artifact` to satisfy the invariant that referenced + # artifacts are uploaded+acked BEFORE the receipt that names them -- the adapter is threaded in + # once `run_job` has built it, rather than the CallRunner reaching for a global. + build_call_runner: Callable[["OutboundAdapter"], CallRunner] = field( + default=lambda adapter: NotWiredCallRunner() + ) + build_world_factory: Callable[[Path], WorldFactory] = field(default=ProcessWorldFactory) + retry_policy: Callable[[], ob.RetryPolicy] = field(default=lambda: ob.RetryPolicy()) + clock: Callable[[], datetime] = field(default=lambda: datetime.now(timezone.utc)) + cancel_path: Path = field(default_factory=lambda: Path(CANCEL_SIGNAL_PATH)) + secrets_path: Path = field(default_factory=lambda: SECRETS_PATH) + flush_window_seconds: float = ob.FLUSH_WINDOW_SECONDS + install_sigterm_handler: Callable[[CancelState], Callable[[], None]] = field( + default=default_install_sigterm_handler + ) + events_spool_dir_name: str = EVENTS_SPOOL_DIR_NAME + scenarios_client_kwargs: dict[str, Any] = field(default_factory=dict) + + def build_events_spool(self, work_directory: Path) -> ob.OutboundSpool: + return ob.OutboundSpool( + work_directory / self.events_spool_dir_name, "events", sequenced=True + ) + + def build_scenarios_client( + self, + capabilities: ob.HostedCapabilities, + transport: ob.Transport, + channel_state: ob.ChannelState, + ) -> ScenariosClient: + return ScenariosClient( + capabilities, transport, channel_state=channel_state, **self.scenarios_client_kwargs + ) + + def peek_secret_values(self) -> tuple[str, ...]: + return peek_secret_values(self.secrets_path) + + +# ================================================================================================= +# Orchestration -- steps 1-8, in order. +# ================================================================================================= + + +async def run_job( + job_path: Path, source: Path, output: Path, *, deps: HostedEntrypointDeps | None = None +) -> int: + """The guest's whole `main()` body. Returns the process exit code (§0.6) — `main()` below is + the only caller that turns this into `SystemExit`, so tests can call this directly and assert + on the return value.""" + deps = deps or HostedEntrypointDeps() + work_directory = output.parent + + # 1. Boot -- capabilities. CapabilitiesError -> exit non-zero-and-NOT-3, no event (v1.3 table): + # there is no channel yet to report a terminal event through. + try: + capabilities = deps.load_capabilities() + except ob.CapabilitiesError as exc: + logger.error("capabilities load failed: %s: %s", exc.code, exc.message) + return EXIT_BOOT_FAILURE + + channel_state = ob.ChannelState() + transport = deps.build_transport() + retry_policy = deps.retry_policy() + events_spool = deps.build_events_spool(work_directory) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, channel_state=channel_state + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state + ) + scenarios_client = deps.build_scenarios_client(capabilities, transport, channel_state) + + adapter = OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=deps.peek_secret_values(), + clock=deps.clock, + flush_window_seconds=deps.flush_window_seconds, + ) + + cancel_state = CancelState(deps.cancel_path) + restore_sigterm = deps.install_sigterm_handler(cancel_state) + # held outside the try so an exception on any path after this line still lets the + # `finally` below close whatever was actually provisioned. + pool: WorldPool | None = None + + def cancel_requested() -> bool: + requested = cancel_state.requested() or adapter.is_fenced + if requested: + # "120s from the cancel signal / TTL / terminal event" -- whichever comes first; + # a cancel/fence observed here starts the clock even though the terminal event (which + # also arms it, unconditionally) may not land until much later. + adapter.arm_flush_window() + return requested + + async def _bounded_close() -> None: + if pool is None: + return + remaining = adapter.deadline() + if remaining is None: + await pool.close() + return + timeout = max(0.0, remaining - time.monotonic()) + try: + await asyncio.wait_for(pool.close(), timeout=timeout) + except asyncio.TimeoutError: + logger.warning( + "pool.close() did not finish within the remaining flush window (%.1fs); " + "WorldPool already latches itself closed on entry to close(), so the top-level " + "finally's own pool.close() call cannot retry the teardown -- the provisioner may " + "be left not fully torn down until close()'s latch ordering changes", timeout, + ) + + async def _finish( + stage: HarnessStage, *, reason: ob.TerminalReason | None = None, + failure: dict[str, Any] | None = None, complete: bool, + scheduler_result: tuple[HostedScheduler, RunResult] | None = None, + ) -> int: + """Terminal event -> drain -> bounded close, in that order, for every path that + reaches a genuine terminal stage -- FAILED (via `_fail`; this now covers the pre-run + failure branches too, not just post-run ones), CANCELED, an aborted RunResult, COMPLETED. + Spending close()'s W-engine teardown time BEFORE a single terminal event is queued is + exactly the inversion this ordering guards against. + + `scheduler_result` is only ever passed by the three call sites reached AFTER + `scheduler.run()` -- pre-run terminals (`_fail`, the boundary `_canceled()` checks) have + no `RunResult` and pass nothing, so this stays a no-op there.""" + await adapter.emit_terminal(stage=stage, reason=reason, failure=failure) + # (outbound-channels.md v1.3 Sequencing): skipped receipts go out AFTER the terminal + # event, never before -- placed here so no return path below can skip this call while + # still delivering the terminal. A fenced result emits nothing further (the scheduler's + # own no-op covers it too; checked here as well so a fenced run never even attempts it). + # Best-effort like every other post-terminal emission in this module: a failure here must + # not undo the terminal already spooled above or change the exit code below. + if scheduler_result is not None: + finished_scheduler, run_result = scheduler_result + if run_result.fenced is None: + try: + # `emit_terminal` above only spools the terminal locally -- flushed to the + # wire here, BEFORE the skipped-receipt pushes below, so a receipt's own + # trailing flush can never deliver the terminal as a side effect and land it + # after that receipt on the wire. Not itself wrapped in the wait_for below -- + # it already threads the same deadline through every retry it makes, and it + # runs first, so its own delivery attempt is never the thing a timeout cuts off. + await adapter.flush_terminal(deadline=adapter.deadline()) + if not adapter.is_fenced: + # `emit_skipped_receipts`/`receipt()` have no deadline plumbing of their + # own (`push()`/`aflush_events()` run with `deadline=None`) -- a + # degraded-but-alive events channel can retry every skipped scenario's + # receipt for the full `RetryPolicy` budget, scaling with how many + # scenarios were cut short and blowing past the flush window the gateway + # tears the sandbox down at. Bounded the same way `_bounded_close` bounds + # `pool.close()`: past the deadline, stop trying and fall through to close. + remaining = adapter.deadline() + timeout = ( + None if remaining is None else max(0.0, remaining - time.monotonic()) + ) + try: + await asyncio.wait_for( + finished_scheduler.emit_skipped_receipts(run_result), + timeout=timeout, + ) + except asyncio.TimeoutError: + logger.warning( + "flush window exhausted before emit_skipped_receipts finished; " + "remaining scenarios' receipts were not sent" + ) + except Exception as exc: # noqa: BLE001 - post-terminal telemetry, never fatal + logger.error("emit_skipped_receipts failed: %s", exc) + # unlinked AFTER the terminal event, not before -- every terminal path shares the + # "secrets are no longer needed past this point" rule, but an unlink failure (a read-only + # or non-owned /run/futureagi) must never cost the one event that proves the job reached a + # terminal state at all. missing_ok=True still no-ops on paths where the provider's own + # §4.4 close() already removed the file; a genuine OSError is logged, not raised -- + # deleting an already-unneeded file is best-effort, not load-bearing. + try: + deps.secrets_path.unlink(missing_ok=True) + except OSError as exc: + logger.warning("secrets.json unlink failed: %s", exc) + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + # the exit code comes from drain()'s own post-hoc fence check (deadline computed AFTER + # emit_terminal, which is what arms the flush window), never a stale pre-drain read. + fenced = await adapter.drain(deadline=adapter.deadline(), complete=complete) + await _bounded_close() + if fenced: + return EXIT_FENCED + # §0.6 v1.14: the terminal was decided but the final drain could not flush it (the events + # channel failed) or the platform permanently rejected the terminal item itself -- exit 0 + # would claim a flush that provably never happened and silently lose the run's evidence. + if adapter.terminal_undelivered: + return EXIT_TERMINAL_UNDELIVERED + return EXIT_OK + + async def _canceled( + *, scheduler_result: tuple[HostedScheduler, RunResult] | None = None, + ) -> int: + return await _finish( + HarnessStage.CANCELED, reason=cancel_state.reason(), complete=False, + scheduler_result=scheduler_result, + ) + + async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, message: str) -> int: + # routed through `_finish` -- terminal event first, pool close (bounded) after, for + # every pre-run failure branch too, not just the post-run ones `_finish` already covered. + return await _finish( + HarnessStage.FAILED, + failure={ + "domain": domain.value, "stage": fail_stage.value, "code": code, "message": message, + }, + complete=True, + ) + + try: + # 1 (cont'd). job.json (§0.2). + try: + job = load_job(job_path) + except Exception as exc: # noqa: BLE001 - a malformed job.json has no typed error to catch + # EXIT_CRASHED (not a FAILED terminal) even though a channel now exists -- a + # malformed job.json means `job.seed`/`job.agent`/etc are not trustworthy enough to + # build a reportable failure from, and every downstream stage assumes a valid `job`. + logger.error("job.json invalid: %s", exc) + return EXIT_CRASHED + + if job.seed is None: + logger.warning("job.seed is null; spine §1 guarantees a concrete integer -- using 0") + job_seed = job.seed if job.seed is not None else 0 + parallelism = resolve_parallelism(job) + secret_purposes = job_secret_purposes(job) + adapter.configure_artifacts(job.artifacts) # level table + budget, now that job.json is known. + + adapter.stage_changed(HarnessStage.VALIDATING_ENVIRONMENT) + await adapter.aflush_events() + + # Bundle authoring is not this module's (see the class docstrings above) -- injected. + try: + manifest, bundle_dir = await asyncio.to_thread( + deps.bundle_source.load, job, source=source, work_directory=work_directory + ) + except BundleUnavailableError as exc: + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_ENVIRONMENT, + code=exc.code, message=exc.message, + ) + + # 2. Preflight -- BEFORE any provision (§2e). `parallelism` is the RAW requested value + # (never clamped), so an out-of-1..8 W fails HERE with `parallelism_out_of_range`, per + # §2e.7, rather than being silently laundered into a valid one. + try: + await asyncio.to_thread( + preflight_bundle, bundle_dir, manifest, parallelism=parallelism, + secret_refs=secret_purposes, + ) + except PreflightError as exc: + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_ENVIRONMENT, + code=exc.code, message=exc.message, + ) + + # cancel/fence check at the post-preflight stage boundary. + if cancel_requested(): + return await _canceled() + + # 4/5. Provision -- ProcessRuntimeProvider, hosted lane never passes + # require_declared_user=False (the provider defaults it True on its own; the local lane's + # opt-out is a construction-site concern, not this module's). Wrapped in the §4.5b + # provider mutex (SerializingProvider) before it ever reaches WorldPool. + provider = SerializingProvider(deps.build_provider()) + pool = WorldPool( + provider, bundle=manifest, source=source, bundle_dir=bundle_dir, + work_directory=work_directory, instances=parallelism, outbound=adapter, + ) + try: + await pool.start() + except (ob.HostedFencedError, ob.HostedAttemptSupersededError): + # defensive -- nothing today routes a channel error through `pool.start()`, but a + # fenced attempt must never fall into the bare `Exception` handler below and get a + # terminal FAILED event synthesized for it. + await _bounded_close() + return EXIT_FENCED + except ob.HostedChannelFailedError as exc: + # `_fail` closes the pool itself now, AFTER the terminal event (via `_finish`) -- + # closing here first was the same close-before-terminal inversion that the terminal -> drain -> close ordering fixes elsewhere. + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + except ProcessRuntimeError as exc: + # §2f's own domain (never the flattened `infrastructure`/"provision_failed" every + # provisioning failure used to get), stage `building_environment` per §2f. + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=_process_runtime_error_domain(exc, manifest), + fail_stage=HarnessStage.BUILDING_ENVIRONMENT, + code=_section_2f_code(exc.code), message=str(exc), + ) + except Exception as exc: # noqa: BLE001 - genuinely untyped -> infrastructure is the honest default + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.INFRASTRUCTURE, fail_stage=HarnessStage.BUILDING_ENVIRONMENT, + code=_section_2f_code("provision_failed"), message=f"provision_failed: {exc}", + ) + + # baseline_frozen + parallelism_degraded from build.json. The whole + # block is guarded -- a malformed build.json value must degrade to a `log`, never kill a + # run that has already provisioned real worlds. + degrade_emitted = False + try: + build_output = await asyncio.to_thread(load_build_output, work_directory) + except WorldFactoryError: + build_output = {} + try: + for store in build_output.get("stores", []): + if store.get("baseline_reference"): + adapter.baseline_frozen( + inputs_digest=str(store.get("inputs_digest", "")), + baseline_ref=str(store.get("baseline_reference", "")), + ) + degrade_reason = build_output.get("degrade_reason") + if degrade_reason: + requested = int(build_output.get("requested_parallelism") or parallelism) + effective = int(build_output.get("effective_parallelism") or 1) + # `ParallelismDegradedPayload` requires `1 <= effective < requested` -- + # `fixed_port` is recorded at `instances == 1` too (provider-side gap), where + # `effective == requested == 1` is not representable as a degrade at all. + if effective < requested: + adapter.parallelism_degraded( + requested=requested, effective=effective, reason=str(degrade_reason) + ) + degrade_emitted = True + else: + await adapter.log( + level="warning", + message=( + f"degrade recorded ({degrade_reason}) with requested==effective==" + f"{requested}; no parallelism_degraded event is representable" + ), + ) + except Exception as exc: # noqa: BLE001 - malformed build.json must never crash a live run + await adapter.log( + level="warning", message=f"build.json degrade/baseline block malformed: {exc}", + ) + # `pool.effective_size` is the ground truth for how many worlds actually exist -- + # if it's short of what was requested and build.json's own `degrade_reason` didn't already + # announce it (a runtime degrade build.json doesn't record), say so loudly rather + # than silently. + if not degrade_emitted and pool.effective_size < parallelism: + await adapter.log( + level="warning", + message=( + f"world pool effective_size={pool.effective_size} < requested " + f"parallelism={parallelism}, but build.json recorded no representable " + "degrade_reason" + ), + ) + await adapter.aflush_events() + + # cancel/fence check at the post-provision stage boundary. + if cancel_requested(): + return await _canceled() + + # 3. Scenario pre-allocation (spine §5 step 3.5). Generation is not this module's; the + # pre-allocation CLIENT (ScenariosClient) is. + adapter.stage_changed(HarnessStage.VALIDATING_SCENARIOS) + await adapter.aflush_events() + world_factory = deps.build_world_factory(work_directory) + try: + scenarios = await deps.scenario_source.build( + job, manifest, scenarios_client, pool=pool, world_factory=world_factory + ) + except (ob.HostedFencedError, ob.HostedAttemptSupersededError): + # `ScenariosClient._post` re-raises these after latching `channel_state` -- a fence + # here must exit 3 with no terminal event, never fall through to the generic handler. + await _bounded_close() + return EXIT_FENCED + except ob.HostedChannelFailedError as exc: + # `ScenariosClient._post` has already latched `channel_state` by the time this branch + # runs -- `emit_terminal` still spools the terminal locally (it never touches the + # network), but the drain that would flush it inherits the same latched channel and can + # never deliver. `_finish` detects exactly this (the terminal's own spool sequence never + # gets acked) and reports it honestly rather than claiming a flush that cannot happen. + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + except (ScenarioSourceNotWired, ScenarioPreallocationError) as exc: + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + + # cancel/fence check at the post-pre-allocation stage boundary. + if cancel_requested(): + return await _canceled() + + # 5/6. Scheduler wiring. + adapter.stage_changed(HarnessStage.RUNNING) + await adapter.aflush_events() + call_runner = deps.build_call_runner(adapter) + scheduler = HostedScheduler( + pool=pool, world_factory=world_factory, call_runner=call_runner, outbound=adapter, + job_seed=job_seed, cancel_requested=cancel_requested, + ) + result: RunResult = await scheduler.run(scenarios) + + # 7. Terminal + exit codes. Terminal -> drain -> close (bounded), never close() first. + if cancel_state.requested(): + return await _canceled(scheduler_result=(scheduler, result)) + if result.aborted is not None: + # v1.14 §5.4 pass-through: `result.aborted.domain`/`.code` carry straight through, not + # flattened to a fixed infrastructure/world_pool_exhausted pair -- the scheduler already + # resolves whether every world failed on the SAME never-retried §2f code (environment + # or agent domain) or a mixed set (`world_pool_exhausted`/`infrastructure`), and this + # just reports that verdict unchanged. + return await _finish( + HarnessStage.FAILED, + failure={ + "domain": result.aborted.domain, "stage": HarnessStage.RUNNING.value, + "code": result.aborted.code, "message": result.aborted.message, + }, + complete=False, + scheduler_result=(scheduler, result), + ) + # `complete: true` only on a genuine, nothing-cut-short COMPLETED terminal. + return await _finish(HarnessStage.COMPLETED, complete=True, scheduler_result=(scheduler, result)) + finally: + if pool is not None: + try: + await pool.close() # idempotent backstop for any path above that missed one. + except Exception: # noqa: BLE001 - a finally must never mask the real exit path + logger.exception("pool.close() failed in the run_job finally backstop") + restore_sigterm() + + +# ================================================================================================= +# CLI -- spine §0 step 5's frozen invocation line: +# `python -m fi.alk.harness.hosted_entrypoint /work/job.json --source /work/source --output +# /work/artifacts` +# ================================================================================================= + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="alk-harness-worker") + parser.add_argument("job", type=Path, help="typed HarnessJob JSON (/work/job.json)") + parser.add_argument("--source", required=True, type=Path, help="/work/source checkout root") + parser.add_argument("--output", required=True, type=Path, help="/work/artifacts") args = parser.parse_args(argv) - return asyncio.run(_run(args.job, args.source, args.output)) + return asyncio.run(run_job(args.job, args.source, args.output)) if __name__ == "__main__": raise SystemExit(main()) + + +__all__ = [ + "CANCEL_SIGNAL_PATH", + "EXIT_BOOT_FAILURE", + "EXIT_CRASHED", + "EXIT_FENCED", + "EXIT_OK", + "EXIT_TERMINAL_UNDELIVERED", + "BundleSource", + "BundleUnavailableError", + "CallRunnerNotWired", + "CancelState", + "DefaultBundleSource", + "HostedEntrypointDeps", + "NotWiredCallRunner", + "NotWiredScenarioSource", + "OutboundAdapter", + "ProcessWorldFactory", + "ScenarioPreallocationError", + "ScenarioSource", + "ScenarioSourceNotWired", + "ScenariosClient", + "SerializingProvider", + "WorldFactoryError", + "install_sigterm_handler", + "job_secret_purposes", + "load_build_output", + "load_job", + "main", + "peek_secret_values", + "resolve_parallelism", + "row_counts_for_capability", + "run_job", +] diff --git a/src/fi/simulate/runtime/spec.py b/src/fi/simulate/runtime/spec.py index 681d19f0..15bab234 100644 --- a/src/fi/simulate/runtime/spec.py +++ b/src/fi/simulate/runtime/spec.py @@ -70,6 +70,9 @@ class RuntimeRequirements(BaseModel): concurrency_weight: int = Field(default=1, ge=1) max_duration_seconds: int = Field(default=300, ge=1) network_policy: str = "live" + # World count for the hosted harness (seam contract §1); the gateway caps + # it at admission, so the model stays permissive beyond ge=1. + parallelism: int = Field(default=1, ge=1) class TimeoutPolicy(BaseModel): diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py new file mode 100644 index 00000000..3a553baf --- /dev/null +++ b/tests/harness/test_hosted_entrypoint.py @@ -0,0 +1,2187 @@ +"""`hosted_entrypoint.py` against in-memory fakes — no real postgres, no real network. + +`asyncio.run` drives every `async def` seam here, matching `test_hosted_scheduler.py`'s own +convention (no pytest-asyncio dependency in this repo). Verification for this file was done by +importing it and calling each `test_*` function directly, not via a `pytest` +invocation. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import random +import stat +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from fi.alk.harness import hosted_entrypoint as he +from fi.alk.harness import outbound as ob +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + EnvironmentBundleV2, + ManagedEngine, + compute_inputs_digest, + seal_bundle_v2, +) +from fi.alk.harness.hosted_scheduler import ( + Call, + CallAborted, + CallOutcome, + HostedScheduler, + ReceiptFailure, + ResultReceipt, + RunResult, +) +from fi.alk.harness.job import ( + AgentConnection, + ArtifactLevel, + ExecutionMode, + HarnessArtifactPolicy, + HarnessJob, + HarnessStage, + RepositorySource, + SourceKind, + SourceVisibility, +) +from fi.alk.harness.process_runtime import ( + EnvironmentRuntime, + ProcessRuntimeError, + RuntimeEndpoint, + RuntimeState, +) +from fi.simulate.runtime.spec import RuntimeRequirements, SecretRef + +SCHEMA_SQL = b"CREATE TABLE riders (id int);\n" +SEED_SQL = b"INSERT INTO riders VALUES (1);\n" +TARGET_PROVIDER_ALIAS = "LIVEKIT_API_KEY" + + +# ================================================================================================= +# Bundle fixture — mirrors test_process_preflight.py's own helper (not imported: this file is +# self-contained per the "touch only your two new files" rule). +# ================================================================================================= + + +def _base_manifest_body() -> dict[str, Any]: + return { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": [], + }, + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], "user": "svc-agent", "depends_on": ["postgres"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", "service": "postgres", "configuration_name": "DATABASE_URL", + }, + }, + "readiness": [], + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64, + }, + "metadata": {}, + } + + +def _write_bundle(root: Path) -> EnvironmentBundleV2: + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL} + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", "migrations": ["db/schema.sql"], "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +def _job( + *, connector: str = "vapi", parallelism: int = 1, + artifacts: HarnessArtifactPolicy | None = None, +) -> HarnessJob: + return HarnessJob( + job_id="job-1", run_id="run-1", execution=ExecutionMode.HOSTED, + source=RepositorySource( + kind=SourceKind.GITHUB, repository="org/repo", visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + ), + agent=AgentConnection( + connector=connector, + secret_refs={ + TARGET_PROVIDER_ALIAS: SecretRef( + manager="platform-vault", key="secret-id", purpose="target_provider" + ) + }, + ), + scenario_count=2, + seed=1234, + runtime=RuntimeRequirements(parallelism=parallelism), + **({"artifacts": artifacts} if artifacts is not None else {}), + ) + + +def _write_job(path: Path, job: HarnessJob) -> None: + path.write_text(job.model_dump_json(), encoding="utf-8") + + +# ================================================================================================= +# Capabilities fixture. +# ================================================================================================= + + +def _capabilities(*, attempt_id: str = "attempt-1") -> ob.HostedCapabilities: + base = f"https://platform.example/simulate/api/harness/attempts/{attempt_id}" + return ob.HostedCapabilities.model_validate( + { + "schema_version": ob.CAPABILITIES_SCHEMA_VERSION, + "job_id": "job-1", + "attempt_id": attempt_id, + "attempt_number": 1, + "fence": "fence-1", + "expires_at": "2999-01-01T00:00:00.000Z", + "token": "bearer-token", + "endpoints": { + "events": f"{base}/events/", + "results": f"{base}/results/", + "artifacts": f"{base}/artifacts/", + "scenarios": f"{base}/scenarios/", + }, + } + ) + + +# ================================================================================================= +# FakeTransport — a minimal in-memory platform. Routes on a URL substring, not a full router: this +# module only needs the four channel shapes, not a general HTTP mock. +# ================================================================================================= + + +@dataclass +class FakeTransport: + fence_after: int | None = None # 1-based call count at which every further call 403s. + fence_on_url_substring: str | None = None # once a URL matches, that call and every later one 403s. + # fences the specific events POST whose batch carries a `type: "terminal"` record -- + # `emit_terminal()`'s own spool append succeeds, so this reproduces the fence landing on the + # network flush that delivers the terminal event, not before it. + fence_on_terminal_event: bool = False + _fenced: bool = False + calls: list[dict[str, Any]] = field(default_factory=list) + event_records: list[dict[str, Any]] = field(default_factory=list) + receipts: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict) + artifacts: dict[str, bytes] = field(default_factory=dict) + manifests: list[dict[str, Any]] = field(default_factory=list) + scenarios_calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + del timeout + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + if self.fence_on_url_substring is not None and self.fence_on_url_substring in url: + self._fenced = True + if ( + self.fence_on_terminal_event + and "/events/" in url + and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + self._fenced = True + if self._fenced or (self.fence_after is not None and len(self.calls) >= self.fence_after): + return ob.TransportResponse( + status_code=403, + body={"error": "fenced", "message": "attempt superseded", "retryable": False}, + headers={}, + ) + if "/events/" in url and method == "POST": + body_bytes = data if isinstance(data, (bytes, bytearray)) else b"" + body = json.loads(body_bytes.decode("utf-8")) if body_bytes else {"events": []} + events = body.get("events", []) + self.event_records.extend(events) + watermark = max((e["sequence"] for e in events), default=0) + return ob.TransportResponse(200, {"acked_through_sequence": watermark, "rejected": []}, {}) + if "/results/" in url and method == "POST" and json_body is not None: + key = (json_body["job_id"], json_body["scenario_key"]) + existed = key in self.receipts + self.receipts[key] = json_body + return ob.TransportResponse(200 if existed else 201, {}, {}) + if url.endswith("/manifest/") and method == "POST" and json_body is not None: + self.manifests.append(json_body) + return ob.TransportResponse(200, {}, {}) + if "/artifacts/" in url and method == "PUT": + digest = url.rstrip("/").rsplit("/", 1)[-1] + payload = data if isinstance(data, (bytes, bytearray)) else b"".join(data) + existed = digest in self.artifacts + self.artifacts[digest] = bytes(payload) + return ob.TransportResponse(200 if existed else 201, {}, {}) + if "/scenarios/" in url and method == "POST" and json_body is not None: + self.scenarios_calls.append((url, json_body)) + if url.endswith("/provision/"): + ids = {key: f"platform-{key}" for key in json_body.get("scenario_keys", [])} + return ob.TransportResponse(200, {"result": {"scenario_ids": ids}}, {}) + return ob.TransportResponse(200, {"result": {"ok": True}}, {}) + return ob.TransportResponse( + 404, {"error": "not_found", "message": f"unmapped route: {url}", "retryable": False}, {} + ) + + def terminal_events(self) -> list[dict[str, Any]]: + return [record for record in self.event_records if record.get("type") == "terminal"] + + +# ================================================================================================= +# Fake provisioner / world / scenario / call runner. +# ================================================================================================= + + +class FakeProvisioner: + name = "fake-process" # mirrors `ProcessRuntimeProvider.name` so passthrough is testable. + + def __init__(self, instances: int = 1, *, always_unhealthy: bool = False) -> None: + self.instances = instances + self.always_unhealthy = always_unhealthy # every healthy() probe fails. + self.provision_calls = 0 + self.reset_calls = 0 + self.healthy_calls = 0 + self.closed = False + self._busy = False + self._runtimes = { + i: EnvironmentRuntime( + runtime_id=f"digest:w{i}", world_index=i, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + for i in range(instances) + } + + async def _serialized(self) -> None: + assert not self._busy, "provider called reentrantly" + self._busy = True + try: + await asyncio.sleep(0) + finally: + self._busy = False + + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract + await self._serialized() + self.provision_calls += 1 + return [self._runtimes[i] for i in range(instances)] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + del work_directory + await self._serialized() + self.reset_calls += 1 + runtime.state = RuntimeState.READY + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + del runtime, work_directory + # v1.12 folds `healthy` into the same non-reentrant set as provision/reset/close -- + # this is the one verb that previously did NOT call `_serialized()`, so a `SerializingProvider` + # gap here would pass silently without it. + await self._serialized() + self.healthy_calls += 1 + return not self.always_unhealthy + + async def close(self, *, work_directory: Path) -> None: + del work_directory + await self._serialized() + self.closed = True + + +class FakeWorld: + def __init__(self, world_index: int, rng: Any) -> None: + self.world_index = world_index + self.rng = rng + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + del table + return {} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + del collection, key + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + del collection, key, changes, by + return 1 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + del collection, key, by + return 1 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + del sql, params + return [] + + def read_only(self) -> "FakeWorld": + return self + + +class FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: Any) -> FakeWorld: + return FakeWorld(runtime.world_index, rng) + + +@dataclass +class FakeSubGoal: + name: str + should_hold: bool + judged: str = "yes" + + def check(self, world: Any, calls: Any) -> object: + del world, calls + return None if self.should_hold else "the agent did not do it" + + +@dataclass +class FakeScenario: + scenario_key: str + scenario_id: str + sub_goals: list[FakeSubGoal] + + def setup(self, world: Any) -> object: + del world + return None + + def ready(self, world: Any) -> object: + del world + return None + + +class FakeCallRunner: + """Uploads a transcript through the adapter BEFORE returning, and can optionally trip the + cancel signal the instant a named scenario starts (for the cancel-mid-run test).""" + + def __init__( + self, + adapter: he.OutboundAdapter, + *, + cancel_path: Path | None = None, + cancel_on_scenario: str | None = None, + cancel_reason: str = "user_canceled", + delay_seconds: float = 0.05, + ) -> None: + self._adapter = adapter + self._cancel_path = cancel_path + self._cancel_on_scenario = cancel_on_scenario + self._cancel_reason = cancel_reason + self._delay_seconds = delay_seconds + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + if self._cancel_path is not None and scenario.scenario_key == self._cancel_on_scenario: + self._cancel_path.write_text( + json.dumps({"reason": self._cancel_reason}), encoding="utf-8" + ) + await asyncio.sleep(self._delay_seconds) + transcript = json.dumps( + [{"speaker_role": "assistant", "content": f"hello from {scenario.scenario_key}"}] + ).encode("utf-8") + artifact_id = await self._adapter.upload_artifact( + transcript, kind=ob.ArtifactKind.TRANSCRIPT, scenario_key=scenario.scenario_key + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=(Call(name="tool", arguments={}, result="ok", ok=True, error="", refused=False, at=0.0),), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=artifact_id, recording_artifacts=(), + ) + + +def _rfc3339(value: datetime) -> str: + return ob.format_rfc3339_millis(value) + + +class FakeScenarioSource: + def __init__(self, scenarios: list[FakeScenario]) -> None: + self._scenarios = scenarios + + async def build( + self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, + world_factory: Any, + ) -> list[FakeScenario]: + del job, bundle, pool, world_factory + await asyncio.to_thread( + scenarios_client.provision, + {"scenario_keys": [s.scenario_key for s in self._scenarios]}, + ) + await asyncio.to_thread(scenarios_client.begin, {"scenario_ids": {}}) + return self._scenarios + + +# ================================================================================================= +# Deps builder. +# ================================================================================================= + + +@dataclass +class Harness: + tmp: Path + work: Path + source: Path + output: Path + job_path: Path + transport: FakeTransport + provisioner: FakeProvisioner + deps: he.HostedEntrypointDeps + + +def _build_harness( + *, + scenarios: list[FakeScenario], + fence_after: int | None = None, + fence_on_url_substring: str | None = None, + fence_on_terminal_event: bool = False, + cancel_on_scenario: str | None = None, + cancel_reason: str = "user_canceled", + corrupt_bundle: Callable[[Path], None] | None = None, + instances: int = 1, + always_unhealthy: bool = False, + parallelism: int = 1, + build_output: dict[str, Any] | None = None, + artifacts: HarnessArtifactPolicy | None = None, +) -> Harness: + tmp = Path(tempfile.mkdtemp(prefix="p10-e2e-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + bundle_dir = work / he.DEFAULT_BUNDLE_DIR_NAME + source.mkdir(parents=True, exist_ok=True) + _write_bundle(bundle_dir) + if corrupt_bundle is not None: + corrupt_bundle(bundle_dir) + + job_path = tmp / "job.json" + _write_job(job_path, _job(parallelism=parallelism, artifacts=artifacts)) + + if build_output is not None: + # `write_build_output` (process_runtime.py) writes here; no test previously did, so + # the baseline_frozen/parallelism_degraded block ran only its + # empty-dict fallback in every prior test. + output.mkdir(parents=True, exist_ok=True) + (output / "build.json").write_text(json.dumps(build_output), encoding="utf-8") + + capabilities = _capabilities() + transport = FakeTransport( + fence_after=fence_after, fence_on_url_substring=fence_on_url_substring, + fence_on_terminal_event=fence_on_terminal_event, + ) + provisioner = FakeProvisioner(instances=instances, always_unhealthy=always_unhealthy) + + cancel_path = tmp / "cancel.json" + + holder: dict[str, he.OutboundAdapter] = {} + + def build_call_runner(adapter: he.OutboundAdapter) -> FakeCallRunner: + holder["adapter"] = adapter + return FakeCallRunner( + adapter, cancel_path=cancel_path, cancel_on_scenario=cancel_on_scenario, + cancel_reason=cancel_reason, + ) + + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + bundle_source=he.DefaultBundleSource(), + scenario_source=FakeScenarioSource(scenarios), + build_transport=lambda: transport, + build_provider=lambda: provisioner, + build_call_runner=build_call_runner, + build_world_factory=lambda work_directory: FakeWorldFactory(), + cancel_path=cancel_path, + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + flush_window_seconds=5.0, + ) + return Harness( + tmp=tmp, work=work, source=source, output=output, job_path=job_path, transport=transport, + provisioner=provisioner, deps=deps, + ) + + +def _run(harness: Harness) -> int: + return asyncio.run(he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps)) + + +def _build_adapter( + transport: FakeTransport, *, extra_secret_values: tuple[str, ...] = () +) -> he.OutboundAdapter: + """Adapter-only fixture for tests that drive `OutboundAdapter` directly (no `run_job`) -- + mirrors `test_redaction_end_to_end_secret_never_crosses_any_channel`'s own inline construction, + factored out for reuse across other adapter-level tests.""" + capabilities = _capabilities() + channel_state = ob.ChannelState() + retry_policy = ob.RetryPolicy() + tmp = Path(tempfile.mkdtemp(prefix="p10-adapter-")) + events_spool = ob.OutboundSpool(tmp / "spool", "events", sequenced=True) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + return he.OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=extra_secret_values, + ) + + +# ================================================================================================= +# Pure-logic unit tests. +# ================================================================================================= + + +def test_resolve_parallelism_reads_the_raw_value_without_clamping() -> None: + # `RuntimeRequirements.parallelism` now exists -- `resolve_parallelism` + # must return it RAW; clamping here would make `parallelism_out_of_range` unreachable. + assert he.resolve_parallelism(_job(parallelism=1)) == 1 + assert he.resolve_parallelism(_job(parallelism=8)) == 8 + # An out-of-range value is preflight's to reject (§2e.7), not this function's to launder -- + # `RuntimeRequirements.parallelism` itself only enforces `ge=1`, so a too-large W passes model + # validation and must still reach `resolve_parallelism` unclamped. + assert he.resolve_parallelism(_job(parallelism=99)) == 99 + + +def test_out_of_range_parallelism_is_rejected_by_preflight_not_clamped() -> None: + # Confirms an out-of-range W reaches a + # `parallelism_out_of_range` preflight rejection (§2e.7), never a silently clamped W=8 run. + async def scenario() -> None: + harness = _build_harness(scenarios=[], parallelism=20) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # rejected before any provision, like §2e. + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["failure"]["code"] == "parallelism_out_of_range" + assert payload["failure"]["domain"] == "environment" + assert payload["failure"]["stage"] == "validating_environment" + + asyncio.run(scenario()) + + +def test_job_secret_purposes_maps_alias_to_purpose() -> None: + job = _job() + assert he.job_secret_purposes(job) == {TARGET_PROVIDER_ALIAS: "target_provider"} + + +def test_peek_secret_values_reads_without_deleting(tmp_path_factory: Path | None = None) -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-secrets-")) + path = tmp / "secrets.json" + path.write_text(json.dumps({"LIVEKIT_API_KEY": "sk-super-secret"}), encoding="utf-8") + values = he.peek_secret_values(path) + assert values == ("sk-super-secret",) + assert path.exists() # non-destructive read -- the provisioner still owns load-and-delete. + + +def test_peek_secret_values_missing_file_is_empty() -> None: + assert he.peek_secret_values(Path("/nonexistent/does-not-exist.json")) == () + + +def test_row_counts_for_capability_returns_the_matching_store() -> None: + build_output = {"stores": [{"capability": "database", "row_counts": {"riders": 3}}]} + assert he.row_counts_for_capability(build_output, "database") == {"riders": 3} + + +def test_row_counts_for_capability_raises_when_the_capability_is_absent() -> None: + build_output = {"stores": [{"capability": "other", "row_counts": {}}]} + try: + he.row_counts_for_capability(build_output, "database") + except he.WorldFactoryError: + pass + else: + raise AssertionError("expected WorldFactoryError") + + +def test_cancel_state_reads_reason_from_file() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-cancel-")) + path = tmp / "cancel.json" + state = he.CancelState(path) + assert state.requested() is False + path.write_text(json.dumps({"reason": "ttl_exceeded"}), encoding="utf-8") + assert state.requested() is True + assert state.reason() is ob.TerminalReason.TTL_EXCEEDED + + +def test_serializing_provider_serializes_concurrent_provision_calls() -> None: + async def scenario() -> None: + fake = FakeProvisioner(instances=1) + wrapped = he.SerializingProvider(fake) + work = Path(tempfile.mkdtemp(prefix="p10-serial-")) + results = await asyncio.gather( + wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), + wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), + ) + # The real, load-bearing check is INSIDE `FakeProvisioner._serialized()` (an `assert not + # self._busy` around a real `await` yield point) -- if `SerializingProvider` let both calls + # run concurrently, that assertion would raise and this whole coroutine would fail instead + # of returning cleanly. `provision_calls == 2` only confirms both eventually ran. + assert fake.provision_calls == 2 + assert all(len(r) == 1 for r in results) + + asyncio.run(scenario()) + + +def test_serializing_provider_serializes_healthy_against_provision() -> None: + # v1.12 folds `healthy` into the SAME non-reentrant set as provision/reset/close -- + # `FakeProvisioner.healthy` is the one verb that previously did not call `_serialized()` + # (see its own definition above), so this is the only test that would have caught a + # `SerializingProvider` that forgot to wrap `healthy()` in its lock. + async def scenario() -> None: + fake = FakeProvisioner(instances=1) + wrapped = he.SerializingProvider(fake) + work = Path(tempfile.mkdtemp(prefix="p10-serial-healthy-")) + runtime = fake._runtimes[0] + await asyncio.gather( + wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), + wrapped.healthy(runtime, work_directory=work), + ) + assert fake.provision_calls == 1 + assert fake.healthy_calls == 1 + + asyncio.run(scenario()) + + +def test_serializing_provider_name_passes_through() -> None: + # §4's `RuntimeProvider` Protocol declares `name: str` -- the wrapper must not hide it. + fake = FakeProvisioner(instances=1) + wrapped = he.SerializingProvider(fake) + assert wrapped.name == "fake-process" + + +def test_scenarios_client_provision_unwraps_the_result_envelope() -> None: + capabilities = _capabilities() + transport = FakeTransport() + client = he.ScenariosClient(capabilities, transport) + result = client.provision({"scenario_keys": ["a", "b"]}) + assert result == {"scenario_ids": {"a": "platform-a", "b": "platform-b"}} + + +def test_scenarios_client_fencing_latches_the_shared_channel_state() -> None: + capabilities = _capabilities() + transport = FakeTransport(fence_after=1) + channel_state = ob.ChannelState() + client = he.ScenariosClient(capabilities, transport, channel_state=channel_state) + try: + client.provision({"scenario_keys": []}) + except ob.HostedFencedError: + pass + else: + raise AssertionError("expected HostedFencedError") + try: + channel_state.check() + except ob.HostedFencedError: + pass + else: + raise AssertionError("channel_state should now be latched for every other channel too") + + +# ================================================================================================= +# Targeted orchestration tests. +# ================================================================================================= + + +def test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event() -> None: + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-boot-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + source.mkdir(parents=True, exist_ok=True) + job_path = tmp / "job.json" + _write_job(job_path, _job()) + + def _raise() -> ob.HostedCapabilities: + raise ob.CapabilitiesError("capabilities_file_missing", "no file") + + deps = he.HostedEntrypointDeps(load_capabilities=_raise) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_BOOT_FAILURE + assert code != he.EXIT_FENCED + assert not (work / he.EVENTS_SPOOL_DIR_NAME).exists() # no channel was ever built. + + asyncio.run(scenario()) + + +def test_preflight_rejection_reaches_a_failed_terminal_event_before_any_provision() -> None: + async def scenario() -> None: + harness = _build_harness( + scenarios=[], + corrupt_bundle=lambda bundle_dir: (bundle_dir / "db" / "schema.sql").unlink(), + ) + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # "BEFORE any provision" + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + assert payload["failure"]["domain"] == "environment" + assert payload["failure"]["stage"] == "validating_environment" + assert payload["failure"]["code"] == "bundle_file_missing" + + asyncio.run(scenario()) + + +def test_hosted_fenced_error_stops_emitting_and_exits_3_with_no_terminal_event() -> None: + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + # Fenced mid-attempt (during the scenario's own receipt push), well after provisioning -- + # proves both halves of the contract: nothing further is ever emitted, AND close() still + # runs (it is unconditional after scheduler.run(), not gated on fencing). + harness = _build_harness(scenarios=scenarios, fence_on_url_substring="/results/") + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True # close() still runs on the way out. + + asyncio.run(scenario()) + + +def test_cancel_mid_run_synthesizes_a_skipped_receipt_for_the_unstarted_scenario() -> None: + async def scenario() -> None: + order: list[str] = [] + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + if ( + "/results/" in url and method == "POST" + and json_body is not None and json_body.get("status") == "skipped" + ): + order.append("skipped_receipt_delivered") + return response + + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + terminals = transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "user_canceled" + statuses = {key[1]: body["status"] for key, body in transport.receipts.items()} + assert statuses.get("first") == "passed" + assert statuses.get("second") == "skipped" + + # outbound-channels.md v1.3 Sequencing ("terminal event -> skipped receipts -> + # manifest") -- the skipped receipt for "second" must reach the platform, and strictly + # after the terminal event, never before or instead of it. + assert order == ["terminal_delivered", "skipped_receipt_delivered"] + + # the "skipped" receipt body (exact) per outbound-channels.md's own six-field list -- + # not just its `status`. + skipped_body = transport.receipts[("job-1", "second")] + assert skipped_body["scenario_attempt"] == 1 + assert skipped_body["world_index"] is None + assert skipped_body["sub_goals"] == [] + assert skipped_body["evaluations"] == [] + assert skipped_body["call"] is None + assert skipped_body["failure"] is None + + # a CANCELED run is cut short -- the manifest must say so. + assert transport.manifests[-1]["complete"] is False + + asyncio.run(scenario()) + + +def test_fenced_run_result_emits_zero_skipped_receipts() -> None: + # `RunResult.fenced` must gate the entrypoint's own call to + # `emit_skipped_receipts` -- checked here at the call site itself, not left to the scheduler's + # own internal no-op. `HostedScheduler.run` is monkeypatched to hand back a fenced `RunResult` + # regardless of how the real run went, since the real `OutboundAdapter` has no path that lets + # a channel fence reach `WorldPool.fenced` (it swallows `HostedFencedError` internally -- + # see `OutboundAdapter._guarded`), so this is the only reliable way to exercise the branch + # end-to-end through `run_job`. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, instances=1) + + fence_exc = ob.HostedFencedError( + ob.ChannelError(ob.ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + original_run = HostedScheduler.run + original_emit = HostedScheduler.emit_skipped_receipts + emit_calls: list[RunResult] = [] + + async def fenced_run(self: HostedScheduler, scns: Any) -> RunResult: + real = await original_run(self, scns) + return RunResult(receipts=real.receipts, aborted=real.aborted, fenced=fence_exc) + + async def counting_emit(self: HostedScheduler, result: RunResult) -> None: + emit_calls.append(result) + await original_emit(self, result) + + HostedScheduler.run = fenced_run # type: ignore[method-assign] + HostedScheduler.emit_skipped_receipts = counting_emit # type: ignore[method-assign] + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.run = original_run + HostedScheduler.emit_skipped_receipts = original_emit + + assert code == he.EXIT_OK + assert emit_calls == [] + + asyncio.run(scenario()) + + +def test_emit_skipped_receipts_failure_does_not_lose_the_terminal_or_the_exit_code() -> None: + # a failure inside the post-terminal `scheduler.emit_skipped_receipts` call must be + # swallowed locally (matching every other best-effort post-terminal emission in this module), + # never mask the terminal already delivered or flip the exit code. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + + original_emit = HostedScheduler.emit_skipped_receipts + + async def poisoned_emit(self: HostedScheduler, result: RunResult) -> None: + raise RuntimeError("synthetic emit_skipped_receipts failure") + + HostedScheduler.emit_skipped_receipts = poisoned_emit # type: ignore[method-assign] + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.emit_skipped_receipts = original_emit + + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "user_canceled" + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("first") == "passed" + + asyncio.run(scenario()) + + +def test_cancel_mid_run_with_ttl_exceeded_reports_that_reason() -> None: + # the exit-code table's CANCELED branch is only ever exercised with `user_canceled` + # elsewhere in this file -- `ttl_exceeded` is the contract's other legal reason + # (`ob.TerminalReason`) and goes through the exact same `CancelState.reason()` path. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness( + scenarios=scenarios, cancel_on_scenario="first", cancel_reason="ttl_exceeded", + instances=1, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "ttl_exceeded" + + asyncio.run(scenario()) + + +def test_malformed_job_json_exits_crashed_with_no_terminal_event() -> None: + # EXIT_CRASHED -- a malformed job.json exits non-zero with no channel of its own + # kind (capabilities load already succeeded, but nothing typed can describe the failure). + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-badjob-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + source.mkdir(parents=True, exist_ok=True) + job_path = tmp / "job.json" + job_path.write_text("{this is not valid json", encoding="utf-8") + + capabilities = _capabilities() + transport = FakeTransport() + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + build_transport=lambda: transport, + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + ) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_CRASHED + assert code != he.EXIT_FENCED + assert transport.terminal_events() == [] + + asyncio.run(scenario()) + + +def test_world_pool_exhaustion_reaches_a_failed_terminal_world_pool_exhausted() -> None: + # `RunResult.aborted` -> a terminal FAILED with domain `infrastructure`, stage + # `running`, code `world_pool_exhausted`. Driven for real: the single world never passes its + # `healthy()` probe, so `WorldPool.lease()` exhausts its reconcile budget and raises + # `NoWorldsAvailable`, which `HostedScheduler.run()` turns into `RunResult.aborted`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, always_unhealthy=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + assert payload["failure"]["domain"] == "infrastructure" + assert payload["failure"]["stage"] == "running" + assert payload["failure"]["code"] == "world_pool_exhausted" + assert harness.provisioner.closed is True + + asyncio.run(scenario()) + + +def test_fence_landing_on_the_final_drain_still_exits_fenced() -> None: + # a fence that 403s specifically the events + # POST carrying the terminal event (not an earlier one) must still exit 3 with no terminal + # event DELIVERED, never a stale pre-drain fence check that misses it. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, fence_on_terminal_event=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True + + asyncio.run(scenario()) + + +def test_fence_from_scenarios_client_exits_fenced_not_crashed() -> None: + # `ScenariosClient._post` re-raises `HostedFencedError` after latching -- + # this must reach `run_job`'s typed handler around `scenario_source.build()`, not fall through + # to a bare `except Exception` (exit 1, and the world pool leaked). + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1, fence_on_url_substring="/scenarios/") + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True # the pool must not leak on this path either. + + asyncio.run(scenario()) + + +def test_scenarios_channel_uses_bearer_auth_never_api_key() -> None: + # outbound-channels.md calls out bearer + X-Harness-Fence by name for pre-allocation -- + # never `X-Api-Key`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + scenarios_calls = [call for call in harness.transport.calls if "/scenarios/" in call["url"]] + assert scenarios_calls, "expected at least one call against endpoints.scenarios" + for call in scenarios_calls: + assert call["headers"].get("Authorization", "").startswith("Bearer ") + assert "X-Api-Key" not in call["headers"] + assert "X-Harness-Fence" in call["headers"] + + asyncio.run(scenario()) + + +def test_process_world_factory_raises_when_no_postgres_endpoint() -> None: + # `ProcessWorldFactory`/`_find_postgres_endpoint` had zero coverage -- only the pure + # `row_counts_for_capability` helper was unit-tested. + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-endpoint-")) + factory = he.ProcessWorldFactory(tmp) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + try: + await factory.create(runtime, rng=random.Random(0)) + except he.WorldFactoryError: + pass + else: + raise AssertionError("expected WorldFactoryError for a runtime with no postgres endpoint") + + asyncio.run(scenario()) + + +def test_process_world_factory_raises_when_build_json_has_no_matching_store() -> None: + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-store-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": [{"capability": "other", "row_counts": {}}]}), encoding="utf-8" + ) + factory = he.ProcessWorldFactory(tmp) + endpoint = RuntimeEndpoint( + capability="database", protocol="postgres", address="postgresql://u:p@localhost/db", + ) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={"database": endpoint}, + ) + try: + await factory.create(runtime, rng=random.Random(0)) + except he.WorldFactoryError: + pass + else: + raise AssertionError("expected WorldFactoryError for a build.json with no matching store") + + asyncio.run(scenario()) + + +def test_build_json_two_stores_emit_two_baseline_frozen_events() -> None: + # Each store in build.json's stores list gets its own baseline_frozen event, not just the first. + async def scenario() -> None: + build_output = { + "stores": [ + { + "capability": "database", "baseline_reference": "ref-database", + "inputs_digest": "digest-database", + }, + { + "capability": "cache", "baseline_reference": "ref-cache", + "inputs_digest": "digest-cache", + }, + ], + } + harness = _build_harness(scenarios=[], instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + baseline_events = [ + record for record in harness.transport.event_records + if record.get("type") == "baseline_frozen" + ] + assert len(baseline_events) == 2 + refs = {event["payload"]["baseline_ref"] for event in baseline_events} + assert refs == {"ref-database", "ref-cache"} + + asyncio.run(scenario()) + + +def test_build_json_degrade_payload_matches_the_recorded_values() -> None: + # `parallelism_degraded`'s payload must mirror build.json's own requested/effective/reason values exactly. + async def scenario() -> None: + build_output = { + "requested_parallelism": 2, "effective_parallelism": 1, + "degrade_reason": "conformance_gate_failed", + } + harness = _build_harness(scenarios=[], instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + degrade_events = [ + record for record in harness.transport.event_records + if record.get("type") == "parallelism_degraded" + ] + assert len(degrade_events) == 1 + payload = degrade_events[0]["payload"] + assert payload == {"requested": 2, "effective": 1, "reason": "conformance_gate_failed"} + + asyncio.run(scenario()) + + +def test_build_json_fixed_port_at_w1_does_not_crash() -> None: + # `requested == effective == 1` with + # `degrade_reason: fixed_port` is not representable as a `parallelism_degraded` event + # (`1 <= effective < requested` fails); this must degrade to a `log`, never crash the run. + async def scenario() -> None: + build_output = { + "requested_parallelism": 1, "effective_parallelism": 1, "degrade_reason": "fixed_port", + } + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.closed is True # the pool must not be orphaned by a crash. + degrade_events = [ + record for record in harness.transport.event_records + if record.get("type") == "parallelism_degraded" + ] + assert degrade_events == [] # not representable -- a log event carries it instead. + log_events = [ + record for record in harness.transport.event_records if record.get("type") == "log" + ] + assert any("fixed_port" in record["payload"]["message"] for record in log_events) + # a substring shared with the pydantic error text the blanket `except Exception` + # would ALSO produce if the `effective < requested` guard were reverted -- this is the one + # phrase that only the guard's own `else` branch ever writes, so it is what actually tells + # the guard apart from the catch-all swallowing a ValidationError. + assert any( + "no parallelism_degraded event is representable" in record["payload"]["message"] + for record in log_events + ) + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + +def test_e2e_two_scenarios_one_pass_one_fail_reaches_completed_and_exits_0() -> None: + async def scenario() -> None: + scenarios = [ + FakeScenario("passing", "platform-passing", [FakeSubGoal("holds", True)]), + FakeScenario("failing", "platform-failing", [FakeSubGoal("holds", False)]), + ] + harness = _build_harness(scenarios=scenarios, instances=1) + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["reason"] is None + assert payload["failure"] is None + assert payload["scenario_counts"] == {"passed": 1, "failed": 1, "errored": 0, "skipped": 0} + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed", "failing": "failed"} + + # both receipts reference an already-uploaded, already-acked transcript artifact. + for key, body in harness.transport.receipts.items(): + del key + transcript = body["call"]["transcript_artifact"] + assert transcript is not None + assert transcript.split(":", 1)[1] in harness.transport.artifacts + + assert len(harness.transport.manifests) >= 1 + assert harness.transport.manifests[-1]["complete"] is True + assert len(harness.transport.manifests[-1]["entries"]) == 2 + + assert harness.provisioner.provision_calls >= 1 + assert harness.provisioner.closed is True + + # Scenario pre-allocation (item 3) actually ran against endpoints.scenarios. + assert any(url.endswith("/provision/") for url, _ in harness.transport.scenarios_calls) + assert any(url.endswith("/begin/") for url, _ in harness.transport.scenarios_calls) + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests closing mutation-honesty gaps -- cases where mutating the guarded code paths +# would have passed the suite unnoticed. +# ================================================================================================= + + +def test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped() -> None: + # deleting the top-level `finally: pool.close()` backstop passed 29/29 -- every prior + # test's exception path had an explicit close ahead of it. `MemoryError` matches none of + # `run_job`'s typed handlers around `scenario_source.build()`, so it propagates straight past + # the `finally` with no explicit close anywhere on this path -- only the backstop can close it. + async def scenario() -> None: + class ExplodingScenarioSource: + async def build(self, job, bundle, scenarios_client, *, pool, world_factory): + del job, bundle, scenarios_client, pool, world_factory + raise MemoryError("boom") + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = ExplodingScenarioSource() + raised = False + try: + await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + except MemoryError: + raised = True + assert raised, "expected the untyped exception to propagate past run_job" + assert harness.provisioner.closed is True # only the finally backstop could have done this + + asyncio.run(scenario()) + + +def test_process_runtime_error_maps_to_the_closed_2f_domain_table() -> None: + # deleting the whole `except ProcessRuntimeError` clause passed 29/29 because + # `ProcessRuntimeError` never appeared anywhere in the suite -- the §2f domain map + # was unexecuted. Drives four real codes through `pool.start()` and checks each domain. + async def run_case(code: str, process: str | None, expected_domain: str) -> None: + class RaisingProvisioner(FakeProvisioner): + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract, instances + raise ProcessRuntimeError("build", code, "synthetic failure", process=process) + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.build_provider = lambda: RaisingProvisioner(instances=1) + result = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert result == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == expected_domain + assert failure["stage"] == "building_environment" + + asyncio.run(run_case("build_failed", None, "agent")) + asyncio.run(run_case("seed_failed", None, "environment")) + asyncio.run(run_case("spawn_failed", "postgres", "infrastructure")) # managed process + asyncio.run(run_case("spawn_failed", "agent", "agent")) # source process + + +def test_finish_emits_the_terminal_before_closing_the_pool() -> None: + # moving `_bounded_close()` before `emit_terminal()` inside `_finish` passed 29/29 -- + # nothing recorded the RELATIVE order of the two. This records both against one shared timeline. + async def scenario() -> None: + order: list[str] = [] + tmp = Path(tempfile.mkdtemp(prefix="p10-order-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + bundle_dir = work / he.DEFAULT_BUNDLE_DIR_NAME + source.mkdir(parents=True, exist_ok=True) + _write_bundle(bundle_dir) + job_path = tmp / "job.json" + _write_job(job_path, _job(parallelism=1)) + capabilities = _capabilities() + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + return response + + class OrderTrackingProvisioner(FakeProvisioner): + async def close(self, *, work_directory: Path) -> None: + await super().close(work_directory=work_directory) + order.append("pool_closed") + + transport = OrderTrackingTransport() + provisioner = OrderTrackingProvisioner(instances=1) + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + bundle_source=he.DefaultBundleSource(), + scenario_source=FakeScenarioSource([]), + build_transport=lambda: transport, + build_provider=lambda: provisioner, + build_world_factory=lambda work_directory: FakeWorldFactory(), + cancel_path=tmp / "cancel.json", + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + flush_window_seconds=5.0, + ) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_OK + assert order == ["terminal_delivered", "pool_closed"] + + asyncio.run(scenario()) + + +def test_redaction_end_to_end_secret_never_crosses_any_channel() -> None: + # dropping the `extra_secret_values` threading (event_builder AND build_result_receipt), + # or the two pre-existing `redact_outbound_text` calls (world_unhealthy.cause, + # receipt.failure.message), all passed 29/29 -- nothing pinned redaction on the wire. Drives the + # secret through a log message, world_unhealthy, a terminal failure, and a receipt failure -- + # the free-text fields the contract names -- and reads what actually reached the transport. + async def scenario() -> None: + secret = "sk-live-eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + capabilities = _capabilities() + transport = FakeTransport() + channel_state = ob.ChannelState() + retry_policy = ob.RetryPolicy() + tmp = Path(tempfile.mkdtemp(prefix="p10-redact-")) + events_spool = ob.OutboundSpool(tmp / "spool", "events", sequenced=True) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, + channel_state=channel_state, + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + adapter = he.OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=(secret,), + ) + + await adapter.log(level="error", message=f"boom: {secret}") + await adapter.world_unhealthy(world_index=0, cause=f"probe failed: {secret}") + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", + message=f"failed calling out with {secret}", + ), + ) + ) + await adapter.emit_terminal( + stage=HarnessStage.FAILED, + failure={ + "domain": "infrastructure", "stage": "building_environment", + "code": "provision_failed", "message": f"connection failed: {secret}", + }, + ) + await adapter.drain(complete=True) + + for record in transport.event_records: + assert secret not in json.dumps(record) + for body in transport.receipts.values(): + assert secret not in json.dumps(body) + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any("***" in r["payload"]["message"] for r in log_events) + world_unhealthy_events = [ + r for r in transport.event_records if r.get("type") == "world_unhealthy" + ] + assert any("***" in r["payload"]["cause"] for r in world_unhealthy_events) + terminal = transport.terminal_events()[0] + assert "***" in terminal["payload"]["failure"]["message"] + receipt_body = transport.receipts[("job-1", "s1")] + assert "***" in receipt_body["failure"]["message"] + + asyncio.run(scenario()) + + +def test_drain_loops_past_a_backlog_larger_than_one_batch_and_still_delivers_the_terminal() -> None: + # A backlog bigger + # than one `EVENTS_MAX_BATCH` (100) previously stranded the terminal event, the highest + # sequence, while still exiting 0. A call runner that logs 260 chatter events before returning + # reproduces the same shape end to end. + async def scenario() -> None: + class ChattyCallRunner: + def __init__(self, adapter: he.OutboundAdapter, *, log_count: int) -> None: + self._adapter = adapter + self._log_count = log_count + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + for i in range(self._log_count): + await self._adapter.log(level="info", message=f"chatter {i}") + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.build_call_runner = lambda adapter: ChattyCallRunner(adapter, log_count=260) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # must not be stranded behind the backlog + + chatter_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and "chatter" in record["payload"].get("message", "") + ] + assert len(chatter_logs) == 260 # the WHOLE backlog drained, not just the first batch + + asyncio.run(scenario()) + + +def test_call_aborted_with_no_ended_at_still_produces_a_receipt() -> None: + # `CallAborted.partial.ended_at` is legitimately `None` (the + # call started but never finished). Before the fix, `build_result_receipt` raised inside + # `OutboundAdapter.receipt()` (outbound.CallSummary.ended_at is a required str), swallowed by + # `HostedScheduler._emit`'s blanket `except Exception` -- the scenario reached the wire with NO + # receipt at all despite `terminal.scenario_counts` claiming one `errored`. + async def scenario() -> None: + class AbortingCallRunner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + now = _rfc3339(datetime.now(timezone.utc)) + raise CallAborted( + "ran out of time before the call finished", + partial=CallOutcome( + calls=(), turns=1, started_at=now, ended_at=None, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.build_call_runner = lambda adapter: AbortingCallRunner() + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("s1") == "errored" + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["call"] is not None + assert receipt_body["call"]["ended_at"] == receipt_body["call"]["started_at"] + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["scenario_counts"]["errored"] == 1 + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests: artifact-level admission coverage, terminal-delivery/receipt-rejection/ +# message-capping edge cases, and mutation-survivor gaps around cancellation and the CANCELED manifest. +# ================================================================================================= + + +def test_metadata_only_artifact_level_refuses_transcript_upload_end_to_end() -> None: + # `_ARTIFACT_LEVEL_FORBIDDEN_KINDS` had zero suite coverage -- emptying the table passed + # every test. Drives a real `metadata-only` job through the default transcript-uploading call + # runner and reads what actually reached the transport. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, + artifacts=HarnessArtifactPolicy(level=ArtifactLevel.METADATA_ONLY), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.transport.artifacts == {} # zero bytes reached the transport + + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["status"] == "passed" # a refused upload must not error the scenario + assert receipt_body["call"]["transcript_artifact"] is None + + log_events = [r for r in harness.transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "kind=transcript" in r["payload"]["message"] + and "forbidden at level=metadata-only" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_traces_artifact_level_refuses_recording_upload_end_to_end() -> None: + # A second shape at a different level -- `traces` allows transcripts but forbids + # recordings. A custom call runner uploads both so the table's per-kind behaviour is visible, + # not just its per-level all-or-nothing behaviour. + async def scenario() -> None: + class RecordingCallRunner: + def __init__(self, adapter: he.OutboundAdapter) -> None: + self._adapter = adapter + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + transcript_id = await self._adapter.upload_artifact( + b"transcript-bytes", kind=ob.ArtifactKind.TRANSCRIPT, + scenario_key=scenario.scenario_key, + ) + recording_id = await self._adapter.upload_artifact( + b"recording-bytes", kind=ob.ArtifactKind.RECORDING_COMBINED, + scenario_key=scenario.scenario_key, + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=transcript_id, + recording_artifacts=(recording_id,) if recording_id else (), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, + artifacts=HarnessArtifactPolicy(level=ArtifactLevel.TRACES), + ) + harness.deps.build_call_runner = lambda adapter: RecordingCallRunner(adapter) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["status"] == "passed" + assert receipt_body["call"]["transcript_artifact"] is not None # traces allows transcripts + assert receipt_body["call"]["recording_artifacts"] == [] # recordings refused at traces + + transcript_digest = receipt_body["call"]["transcript_artifact"].split(":", 1)[1] + assert set(harness.transport.artifacts) == {transcript_digest} # the recording never uploaded + + log_events = [r for r in harness.transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "kind=recording_combined" in r["payload"]["message"] + and "forbidden at level=traces" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_secrets_unlink_failure_after_the_terminal_does_not_lose_the_terminal_event() -> None: + # A non-writable secrets directory must not cost the terminal event -- the + # unlink now runs AFTER `emit_terminal` and is wrapped, so an OSError there is logged, not + # raised past `run_job`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + guard_dir = Path(tempfile.mkdtemp(prefix="p10-ro-")) + secrets_path = guard_dir / "secrets.json" + secrets_path.write_text('{"A": "x"}', encoding="utf-8") + os.chmod(guard_dir, stat.S_IRUSR | stat.S_IXUSR) # read-only directory -> unlink raises + harness.deps.secrets_path = secrets_path + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + finally: + os.chmod(guard_dir, stat.S_IRWXU) + + asyncio.run(scenario()) + + +def test_events_channel_dying_on_the_final_drain_exits_terminal_undelivered() -> None: + # The events channel dies specifically on the flush that carries the terminal + # (sustained 5xx exhausts the retry budget) -- exit 0 would falsely claim a flush that never + # happened. `terminal_undelivered` catches this via the spool watermark never reaching the + # terminal's own sequence. + async def scenario() -> None: + class DyingOnTerminalTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if ( + method == "POST" and "/events/" in url + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + return ob.TransportResponse( + 500, {"error": "server_error", "message": "boom", "retryable": True}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = DyingOnTerminalTransport() + harness.deps.build_transport = lambda: transport + harness.deps.retry_policy = lambda: ob.RetryPolicy( + initial_backoff_seconds=0.0, max_backoff_seconds=0.0, max_attempts=3, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_TERMINAL_UNDELIVERED + assert transport.terminal_events() == [] # never actually reached the platform + + asyncio.run(scenario()) + + +def test_platform_rejecting_the_terminal_event_exits_terminal_undelivered() -> None: + # A different shape: the platform permanently rejects the terminal item itself (a per-item rejection + # inside an otherwise-200 response), distinct from a dead channel. + async def scenario() -> None: + class RejectingTerminalTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if method == "POST" and "/events/" in url and isinstance(data, (bytes, bytearray)): + body = json.loads(bytes(data).decode("utf-8")) + events = body.get("events", []) + terminal = [e for e in events if e.get("type") == "terminal"] + if terminal: + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + keep = [e for e in events if e.get("type") != "terminal"] + self.event_records.extend(keep) + rejected = [ + {"sequence": e["sequence"], "code": "payload_invalid", "message": "nope"} + for e in terminal + ] + watermark = max((e["sequence"] for e in events), default=0) + return ob.TransportResponse( + 200, {"acked_through_sequence": watermark, "rejected": rejected}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = RejectingTerminalTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_TERMINAL_UNDELIVERED + assert transport.terminal_events() == [] # rejected, never landed as a delivered record + + asyncio.run(scenario()) + + +def test_receipt_rejection_by_the_platform_is_logged() -> None: + # `ResultsClient.push()` returns `ReceiptPushResult(error=...)` on a permanent rejection + # rather than raising -- nothing inspected the return value before this fix, so the contract's + # "guest logs, no retry" obligation for e.g. 409 receipt_conflict went unmet. + async def scenario() -> None: + class RejectingResultsTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if method == "POST" and "/results/" in url and json_body is not None: + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + return ob.TransportResponse( + 409, + {"error": "receipt_conflict", "message": "digest mismatch", "retryable": False}, + {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = RejectingResultsTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "s1" in r["payload"]["message"] + and "receipt_conflict" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_receipt_failure_message_is_capped_like_the_terminal_message() -> None: + # `receipt().failure.message` was uncapped while `emit_terminal` caps at 4KB -- an + # oversized receipt failure message is the most plausible route into a 413 (the same silent-loss + # shape as a rejected receipt). Adapter-level: no `run_job` needed for a pure formatting check. + async def scenario() -> None: + transport = FakeTransport() + adapter = _build_adapter(transport) + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", message="y" * 20_000, + ), + ) + ) + body = transport.receipts[("job-1", "s1")] + message = body["failure"]["message"] + assert len(message) <= he._TERMINAL_FAILURE_MESSAGE_MAX_CHARS + assert message.endswith("…[truncated]") + + asyncio.run(scenario()) + + +def test_cancel_before_provision_skips_provisioning_entirely() -> None: + # The pre-provision cancel checkpoint (`hosted_entrypoint.py:1342` area) had no + # test driving a cancel signal written BEFORE `run_job` starts -- disabling all three + # checkpoints still passed the whole suite. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.cancel_path.write_text( + json.dumps({"reason": "ttl_exceeded"}), encoding="utf-8" + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # canceled before provisioning ever ran + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "ttl_exceeded" + + # a CANCELED run is cut short -- the manifest must say so. + assert harness.transport.manifests[-1]["complete"] is False + + asyncio.run(scenario()) + + +def test_evaluation_wire_coerces_an_int_score_to_float() -> None: + # `_evaluation_wire`'s `float(score)` coercion had no direct unit test -- an + # int score digest-mismatches `build_result_receipt`'s own re-derivation and silently drops the + # whole receipt. + class IntScoreEvaluation: + kind = "metric" + name = "accuracy" + score = 1 + reason = "int not float" + + wire = he._evaluation_wire(IntScoreEvaluation()) + assert wire["score"] == 1.0 + assert isinstance(wire["score"], float) + + +def test_receipt_nulls_an_unacked_transcript_artifact_and_logs_it() -> None: + # A receipt naming an artifact id this adapter never uploaded (and therefore + # never acked) must ship `null`/`[]` on the wire, not the un-acked id -- the platform would 422 + # (`artifact_unknown`) the whole receipt otherwise. + async def scenario() -> None: + transport = FakeTransport() + adapter = _build_adapter(transport) + + class UnackedCall: + started_at = "2026-01-01T00:00:00.000Z" + ended_at = "2026-01-01T00:00:01.000Z" + duration_ms = 1000 + turns = 1 + transcript_artifact = "sha256:" + "a" * 64 # never uploaded through this adapter + recording_artifacts = ("sha256:" + "b" * 64,) + + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="passed", sub_goals=(), evaluations=(), call=UnackedCall(), failure=None, + ) + ) + body = transport.receipts[("job-1", "s1")] + assert body["call"]["transcript_artifact"] is None + assert body["call"]["recording_artifacts"] == [] + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" and "un-acked transcript" in r["payload"]["message"] + for r in log_events + ) + assert any( + r["payload"]["level"] == "error" and "un-acked recording" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_pre_run_provision_failure_emits_the_terminal_before_closing_the_pool() -> None: + # `test_finish_emits_the_terminal_before_...` + # only drives the COMPLETED path -- a pre-run failure (`_fail` -> `_finish`) is a structurally + # different entry into `_finish` and needs its own ordering check. + async def scenario() -> None: + order: list[str] = [] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + return response + + class FailingProvisioner(FakeProvisioner): + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract, instances + raise ProcessRuntimeError("build", "build_failed", "synthetic", process="agent") + + async def close(self, *, work_directory: Path) -> None: + await super().close(work_directory=work_directory) + order.append("pool_closed") + + harness = _build_harness(scenarios=[], instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + harness.deps.build_provider = lambda: FailingProvisioner(instances=1) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert order[:2] == ["terminal_delivered", "pool_closed"] + terminals = transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["failure"]["domain"] == "agent" + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests -- the wire retargeted the fence-on-final-drain test to the pre-drain +# check; flush_terminal's and drain()'s bounded loops now cover for each other; the +# post-terminal wire block had no deadline. +# ================================================================================================= + + +def test_fence_on_the_manifest_push_still_exits_fenced_with_the_terminal_already_delivered() -> None: + # `flush_terminal` now delivers the terminal-carrying POST ahead of `drain()`, so a fence + # on that POST is caught by the PRE-drain `is_fenced` check in `_finish`, not the post-drain + # `if fenced: return EXIT_FENCED` line -- `test_fence_landing_on_the_final_ + # drain_still_exits_fenced` no longer exercises that check for the reason its own comment + # claims. A fence that only starts on `/manifest/` lands strictly inside `drain()` itself + # (drain()'s own `push_manifest` call), after a clean `flush_terminal`, so `drain()`'s return + # value -- and therefore only the post-drain check -- is what decides the exit code here. Also + # pins the observation that a fenced attempt can still have already delivered its terminal, + # deliberately rather than by accident. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, fence_on_url_substring="/manifest/", + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # delivered before the fence ever landed + assert terminals[0]["payload"]["stage"] == "completed" + assert harness.provisioner.closed is True + + asyncio.run(scenario()) + + +def test_drain_alone_delivers_a_pre_run_backlog_larger_than_one_batch() -> None: + # On a pre-run terminal (`_fail` -> `_finish` with no `scheduler_result`), the wire's + # `flush_terminal` is never called at all -- `drain()`'s own bounded loop is the ONLY delivery + # mechanism for whatever accumulated beforehand. `test_drain_loops_past_a_backlog_...` only + # drives a post-run COMPLETED path, where `flush_terminal` already drains everything first and + # masks a collapsed `drain()` loop. A `ScenarioSource` that logs a large pre-run backlog before + # raising a typed pre-allocation failure reproduces the shape end to end, on the one path where + # collapsing `drain()`'s loop to a single flush cannot be covered for by anything else. + async def scenario() -> None: + class ChattyPreRunScenarioSource: + def __init__(self, *, chatter_count: int) -> None: + self._chatter_count = chatter_count + + async def build( + self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, + pool: Any, world_factory: Any, + ) -> list[FakeScenario]: + del job, bundle, scenarios_client, world_factory + adapter = pool._outbound # no adapter seam on ScenarioSource itself + for i in range(self._chatter_count): + await adapter.log(level="info", message=f"pre-run chatter {i}") + raise he.ScenarioPreallocationError(None) + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = ChattyPreRunScenarioSource(chatter_count=260) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # must not be stranded behind the pre-run backlog + assert terminals[0]["payload"]["stage"] == "failed" + + chatter_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and "pre-run chatter" in record["payload"].get("message", "") + ] + assert len(chatter_logs) == 260 # the whole backlog drained, not just the first batch + + asyncio.run(scenario()) + + +def test_flush_terminal_alone_must_deliver_the_terminal_before_a_skipped_receipt_under_backlog() -> None: + # The converse of the previous test -- on a cancel-mid-run path, `flush_terminal` is the + # ONLY thing standing between "terminal not yet on the wire" and `emit_skipped_receipts` + # pushing a receipt straight to `/results/` (that push is not gated on event delivery at all). + # A backlog bigger than one batch, queued before the cancel is even noticed, means a collapsed + # `flush_terminal` loop delivers only the first batch and leaves the terminal pending -- the + # skipped receipt for "second" then reaches the platform BEFORE the terminal does, even though + # `drain()`'s own (intact) loop mops up the rest a moment later. Ordering, not delivery count, + # is what only `flush_terminal`'s loop can guarantee here. + async def scenario() -> None: + order: list[str] = [] + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + if ( + "/results/" in url and method == "POST" + and json_body is not None and json_body.get("status") == "skipped" + ): + order.append("skipped_receipt_delivered") + return response + + class ChattyCancelingCallRunner: + def __init__( + self, adapter: he.OutboundAdapter, *, cancel_path: Path, cancel_on_scenario: str, + chatter_count: int, + ) -> None: + self._adapter = adapter + self._cancel_path = cancel_path + self._cancel_on_scenario = cancel_on_scenario + self._chatter_count = chatter_count + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + if scenario.scenario_key == self._cancel_on_scenario: + for i in range(self._chatter_count): + await self._adapter.log(level="info", message=f"chatter {i}") + self._cancel_path.write_text( + json.dumps({"reason": "user_canceled"}), encoding="utf-8" + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ) + + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + harness.deps.build_call_runner = lambda adapter: ChattyCancelingCallRunner( + adapter, cancel_path=harness.deps.cancel_path, cancel_on_scenario="first", + chatter_count=260, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + statuses = {key[1]: body["status"] for key, body in transport.receipts.items()} + assert statuses.get("first") == "passed" + assert statuses.get("second") == "skipped" + + assert order == ["terminal_delivered", "skipped_receipt_delivered"] + + asyncio.run(scenario()) + + +def test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window() -> None: + # `emit_skipped_receipts`/`receipt()` push with `deadline=None` -- a degraded-but-alive + # events channel could previously retry every skipped scenario's receipt for the full + # `RetryPolicy` budget regardless of how much of the flush window was already spent, well past + # the point the gateway tears the sandbox down. A tiny window plus a slow + # `emit_skipped_receipts` reproduces the shape without a real multi-second stall dominating the + # suite: `asyncio.wait_for`'s own cancellation cuts the sleep short well before it would run out. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + harness.deps.flush_window_seconds = 0.2 + + original_emit = HostedScheduler.emit_skipped_receipts + + async def slow_emit(self: HostedScheduler, result: RunResult) -> None: + await asyncio.sleep(2.0) + await original_emit(self, result) + + HostedScheduler.emit_skipped_receipts = slow_emit # type: ignore[method-assign] + started = time.monotonic() + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.emit_skipped_receipts = original_emit + elapsed = time.monotonic() - started + + assert elapsed < 1.5, f"post-terminal wire block was not bounded: {elapsed:.2f}s" + assert code == he.EXIT_OK # the terminal was already delivered before the window ran out + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("first") == "passed" + assert "second" not in statuses # window ran out before the skipped receipt could go + + assert harness.provisioner.closed is True # the finally backstop still closes the pool + + asyncio.run(scenario()) + + +TESTS = [ + test_resolve_parallelism_reads_the_raw_value_without_clamping, + test_out_of_range_parallelism_is_rejected_by_preflight_not_clamped, + test_job_secret_purposes_maps_alias_to_purpose, + test_peek_secret_values_reads_without_deleting, + test_peek_secret_values_missing_file_is_empty, + test_row_counts_for_capability_returns_the_matching_store, + test_row_counts_for_capability_raises_when_the_capability_is_absent, + test_cancel_state_reads_reason_from_file, + test_serializing_provider_serializes_concurrent_provision_calls, + test_serializing_provider_serializes_healthy_against_provision, + test_serializing_provider_name_passes_through, + test_scenarios_client_provision_unwraps_the_result_envelope, + test_scenarios_client_fencing_latches_the_shared_channel_state, + test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event, + test_preflight_rejection_reaches_a_failed_terminal_event_before_any_provision, + test_hosted_fenced_error_stops_emitting_and_exits_3_with_no_terminal_event, + test_cancel_mid_run_synthesizes_a_skipped_receipt_for_the_unstarted_scenario, + test_fenced_run_result_emits_zero_skipped_receipts, + test_emit_skipped_receipts_failure_does_not_lose_the_terminal_or_the_exit_code, + test_cancel_mid_run_with_ttl_exceeded_reports_that_reason, + test_malformed_job_json_exits_crashed_with_no_terminal_event, + test_world_pool_exhaustion_reaches_a_failed_terminal_world_pool_exhausted, + test_fence_landing_on_the_final_drain_still_exits_fenced, + test_fence_from_scenarios_client_exits_fenced_not_crashed, + test_scenarios_channel_uses_bearer_auth_never_api_key, + test_process_world_factory_raises_when_no_postgres_endpoint, + test_process_world_factory_raises_when_build_json_has_no_matching_store, + test_build_json_two_stores_emit_two_baseline_frozen_events, + test_build_json_degrade_payload_matches_the_recorded_values, + test_build_json_fixed_port_at_w1_does_not_crash, + test_e2e_two_scenarios_one_pass_one_fail_reaches_completed_and_exits_0, + test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped, + test_process_runtime_error_maps_to_the_closed_2f_domain_table, + test_finish_emits_the_terminal_before_closing_the_pool, + test_redaction_end_to_end_secret_never_crosses_any_channel, + test_drain_loops_past_a_backlog_larger_than_one_batch_and_still_delivers_the_terminal, + test_call_aborted_with_no_ended_at_still_produces_a_receipt, + test_metadata_only_artifact_level_refuses_transcript_upload_end_to_end, + test_traces_artifact_level_refuses_recording_upload_end_to_end, + test_secrets_unlink_failure_after_the_terminal_does_not_lose_the_terminal_event, + test_events_channel_dying_on_the_final_drain_exits_terminal_undelivered, + test_platform_rejecting_the_terminal_event_exits_terminal_undelivered, + test_receipt_rejection_by_the_platform_is_logged, + test_receipt_failure_message_is_capped_like_the_terminal_message, + test_cancel_before_provision_skips_provisioning_entirely, + test_evaluation_wire_coerces_an_int_score_to_float, + test_receipt_nulls_an_unacked_transcript_artifact_and_logs_it, + test_pre_run_provision_failure_emits_the_terminal_before_closing_the_pool, + test_fence_on_the_manifest_push_still_exits_fenced_with_the_terminal_already_delivered, + test_drain_alone_delivers_a_pre_run_backlog_larger_than_one_batch, + test_flush_terminal_alone_must_deliver_the_terminal_before_a_skipped_receipt_under_backlog, + test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window, +] + + +if __name__ == "__main__": + failures = 0 + for test_fn in TESTS: + started = time.monotonic() + try: + test_fn() + except Exception as exc: # noqa: BLE001 - a direct-invocation runner, not pytest + failures += 1 + print(f"FAIL {test_fn.__name__}: {type(exc).__name__}: {exc}") + else: + print(f"ok {test_fn.__name__} ({time.monotonic() - started:.2f}s)") + print(f"\n{len(TESTS) - failures}/{len(TESTS)} passed") + raise SystemExit(1 if failures else 0) From fb27def28b2bfba150bdfb1ed354110388f2fdd1 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 19:23:58 +0530 Subject: [PATCH 13/20] fix(harness): make the psycopg-absent prober test environment-independent Signed-off-by: khushalsonawat --- tests/harness/test_process_runtime.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py index ddc7aeb0..eff5e836 100644 --- a/tests/harness/test_process_runtime.py +++ b/tests/harness/test_process_runtime.py @@ -1736,16 +1736,16 @@ def make(state: pr.RuntimeState) -> pr.EnvironmentRuntime: # --- default probers: real postgres/http exercise, no fakes -------------------------------------- -def test_default_capability_prober_falls_back_to_tcp_when_psycopg_is_absent() -> None: - """`psycopg` is not installed in this test lane (import-guarded) — the postgres branch must - fall back to a bare TCP probe rather than raising `ImportError`. - - T1, p5-round1-review: the premise itself is asserted, so if `psycopg` is ever installed in - this lane the test fails LOUDLY instead of silently exercising a real-connect code path under - the same name and passing for the wrong reason.""" - assert importlib.util.find_spec("psycopg") is None, ( - "psycopg is installed in this test lane — this test's fallback premise no longer holds" - ) +def test_default_capability_prober_falls_back_to_tcp_when_psycopg_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The postgres branch must fall back to a bare TCP probe rather than raising + `ImportError` on hosts without `psycopg`. Absence is simulated by blocking the + module in `sys.modules` — the prober imports at call time, so the import genuinely + fails and the real fallback branch runs regardless of what this venv has installed + (an environment premise check would flip whenever another lane needs `psycopg` + present).""" + monkeypatch.setitem(sys.modules, "psycopg", None) import socket as socket_module server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) From 0b0ddfce30180a7371d0a57ac128c903688f4ba0 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Tue, 25 Aug 2026 20:44:22 +0530 Subject: [PATCH 14/20] =?UTF-8?q?feat(harness):=20integration=20pass=20?= =?UTF-8?q?=E2=80=94=20provider-resolved=20failure=20domains,=20scenario?= =?UTF-8?q?=20validation,=20single=20domain=20map?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_entrypoint.py | 188 +++++++++--------- src/fi/alk/harness/hosted_scheduler.py | 128 ++++++++----- src/fi/alk/harness/process_preflight.py | 26 ++- src/fi/alk/harness/process_runtime.py | 189 ++++++++++++------- src/fi/alk/harness/world/stores/container.py | 7 +- tests/harness/test_hosted_entrypoint.py | 189 ++++++++++++------- tests/harness/test_hosted_scheduler.py | 41 ++++ tests/harness/test_process_preflight.py | 19 ++ tests/harness/test_world_stores_container.py | 52 +++++ 9 files changed, 559 insertions(+), 280 deletions(-) create mode 100644 tests/harness/test_world_stores_container.py diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 03fa054b..a9273cd3 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -33,7 +33,7 @@ from typing import Any, Callable, Protocol, Sequence from . import outbound as ob -from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, ProcessKind, load_bundle_v2 +from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, load_bundle_v2 from .hosted_scheduler import ( CallOutcome, CallRunner, @@ -56,6 +56,7 @@ ) from .process_preflight import PreflightError, preflight_bundle from .process_runtime import ( + SECTION_2F_DOMAIN, EnvironmentRuntime, ProcessRuntimeError, ProcessRuntimeProvider, @@ -275,61 +276,6 @@ async def build( ) -# ================================================================================================= -# §4.5b -- a single provider mutex serializing every provision/reset/close/healthy call. Explicitly -# this module's duty per the obligations list. `WorldPool` (hosted_scheduler.py) already serializes -# provision/reset/close through its own `_provider_lock`, but NOT `healthy()` (by design — its own -# docstring reads v1.11 §4.5b's non-reentrancy sentence as naming only provision/reset/close); this -# wrapper is the belt-and-suspenders version that holds for all four regardless of what the -# scheduler's own lock covers today, and is safe to layer under it (two distinct `asyncio.Lock` -# objects on a single-threaded event loop cannot deadlock each other). -# ================================================================================================= - - -class SerializingProvider: - def __init__(self, provider: WorldProvisioner) -> None: - self._provider = provider - self._lock = asyncio.Lock() - - @property - def name(self) -> str: - # §4's `RuntimeProvider` Protocol declares `name: str` ("retained for logging only"); - # this wrapper otherwise hides it, so any future `provider.name` read would AttributeError. - return getattr(self._provider, "name", "") - - async def provision( - self, - bundle: Any, - *, - source: Path, - bundle_dir: Path, - work_directory: Path, - contract: Any | None = None, - instances: int = 1, - ) -> list[EnvironmentRuntime]: - async with self._lock: - return await self._provider.provision( - bundle, - source=source, - bundle_dir=bundle_dir, - work_directory=work_directory, - contract=contract, - instances=instances, - ) - - async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: - async with self._lock: - await self._provider.reset(runtime, work_directory=work_directory) - - async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: - async with self._lock: - return await self._provider.healthy(runtime, work_directory=work_directory) - - async def close(self, *, work_directory: Path) -> None: - async with self._lock: - await self._provider.close(work_directory=work_directory) - - # ================================================================================================= # WorldFactory -- real HostedWorld instances, fed by build.json's row counts (never # a partial map). @@ -342,37 +288,25 @@ class WorldFactoryError(RuntimeError): the row counts for that store), never a scenario-code fault.""" -# §2f's closed build/run failure-code table (hosted-execution-seams.md §4.6) -> FailureDomain. -# LOCAL to this module for now -- no module owns this map today. `spawn_failed` is the one code the table itself -# splits by process kind ("infrastructure if a managed engine, agent if source"), so it is resolved -# by a manifest lookup in `_process_runtime_error_domain` below rather than a flat entry here. -_SECTION_2F_DOMAIN: dict[str, FailureDomain] = { - "source_tree_unavailable": FailureDomain.ENVIRONMENT, - "build_failed": FailureDomain.AGENT, - "runtime_unsupported": FailureDomain.ENVIRONMENT, - "depends_on_timeout": FailureDomain.INFRASTRUCTURE, - "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, - "seed_failed": FailureDomain.ENVIRONMENT, - "store_statement_failed": FailureDomain.INFRASTRUCTURE, -} - - -def _process_runtime_error_domain( - exc: ProcessRuntimeError, manifest: EnvironmentBundleV2 -) -> FailureDomain: - if exc.code == "spawn_failed": - for process in manifest.processes: - if process.name == exc.process: - return ( - FailureDomain.INFRASTRUCTURE - if process.kind is ProcessKind.MANAGED - else FailureDomain.AGENT - ) - return FailureDomain.INFRASTRUCTURE # unresolvable process name -- the honest default - return _SECTION_2F_DOMAIN.get(exc.code, FailureDomain.INFRASTRUCTURE) # internal_* etc. +def _process_runtime_error_domain(exc: ProcessRuntimeError) -> FailureDomain: + """v1.15 §2f: the producer (`process_runtime.py`) resolves and carries `domain` at the raise + site -- read it directly rather than re-deriving `spawn_failed`'s managed/source split from + the manifest (the old approach could not tell which process kind failed without one). The + imported `SECTION_2F_DOMAIN` map is a fallback ONLY, for an error that reaches here with no + carried domain -- logged when it fires, matching the scheduler's own rule. + """ + if exc.domain is not None: + return exc.domain + if exc.code in SECTION_2F_DOMAIN: + logger.warning( + "process_runtime error %r crossed the §4 seam with no carried domain; using the §2f " + "fallback map (%s)", exc.code, SECTION_2F_DOMAIN[exc.code].value, + ) + return SECTION_2F_DOMAIN[exc.code] + return FailureDomain.INFRASTRUCTURE # internal_* etc. -- the honest default -_SECTION_2F_CODES: frozenset[str] = frozenset(_SECTION_2F_DOMAIN) | {"spawn_failed"} +_SECTION_2F_CODES: frozenset[str] = frozenset(SECTION_2F_DOMAIN) def _section_2f_code(code: str) -> str: @@ -1244,6 +1178,63 @@ def peek_secret_values(self) -> tuple[str, ...]: return peek_secret_values(self.secrets_path) +# ================================================================================================= +# Scenario-entry validation at fetch (defense against karthik-integration-changes.md K1): the +# Scenario Generation Contract's own model may not carry `scenario_key` (or may hand back some +# other malformed shape) by the time `scenario_source.build()` returns it here, and +# `hosted_scheduler.py` reads `scenario.scenario_key`/`.sub_goals`/`.setup`/`.ready` at its own +# call sites with plain attribute access -- an attribute a pydantic/dataclass model never defined +# raises AttributeError, not a typed failure, deep inside the scheduler with no terminal event and +# a nonzero exit that reads as an infrastructure crash. Checked here with `getattr` (never direct +# attribute access) so a malformed entry is caught at the seam, before the scheduler ever touches +# it -- one bad entry fails the whole job as a typed FAILED terminal instead of crashing the guest. +# ================================================================================================= + +# No closed-vocabulary code names this defect specifically (the §2e/§2f tables are bundle/process +# concerns, not scenario-content ones) -- `scenario_preallocation_failed` is this module's own +# existing code for "the scenario set is not viable for this attempt," already scoped to stage +# `validating_scenarios`, and is reused here rather than inventing a new one. Domain `environment` +# (not `platform_sync`, its other use here): a malformed entry is a deterministic generation-stage +# content defect, not a transport failure, and fails identically on retry. +_SCENARIO_ENTRY_INVALID_CODE = "scenario_preallocation_failed" + + +def _validate_scenario_entry(entry: Any, *, index: int) -> str | None: + """Returns a human-readable defect description, or `None` if `entry` looks usable by + `hosted_scheduler.py`'s `Scenario` Protocol. Every check is a `getattr` with a default, never + a direct attribute/index access -- the whole point is to survive a shape that lacks a field + entirely, not just one that carries a wrong value. + """ + scenario_key = getattr(entry, "scenario_key", None) + if not isinstance(scenario_key, str) or not scenario_key: + return f"scenario[{index}] has no non-empty scenario_key" + label = f"scenario[{index}] ({scenario_key!r})" + if not isinstance(getattr(entry, "scenario_id", None), str): + return f"{label} has no scenario_id" + if not callable(getattr(entry, "setup", None)): + return f"{label} has no callable setup()" + if not callable(getattr(entry, "ready", None)): + return f"{label} has no callable ready()" + sub_goals = getattr(entry, "sub_goals", None) + if not isinstance(sub_goals, Sequence) or isinstance(sub_goals, (str, bytes)): + return f"{label} has no sub_goals sequence" + for goal_index, goal in enumerate(sub_goals): + goal_name = getattr(goal, "name", None) + if not isinstance(goal_name, str) or not goal_name: + return f"{label} sub_goal[{goal_index}] has no non-empty name" + if not callable(getattr(goal, "check", None)): + return f"{label} sub_goal[{goal_index}] ({goal_name!r}) has no callable check()" + return None + + +def _validate_scenarios(scenarios: Sequence[Any]) -> str | None: + for index, entry in enumerate(scenarios): + defect = _validate_scenario_entry(entry, index=index) + if defect is not None: + return defect + return None + + # ================================================================================================= # Orchestration -- steps 1-8, in order. # ================================================================================================= @@ -1479,9 +1470,10 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m # 4/5. Provision -- ProcessRuntimeProvider, hosted lane never passes # require_declared_user=False (the provider defaults it True on its own; the local lane's - # opt-out is a construction-site concern, not this module's). Wrapped in the §4.5b - # provider mutex (SerializingProvider) before it ever reaches WorldPool. - provider = SerializingProvider(deps.build_provider()) + # opt-out is a construction-site concern, not this module's). §4.5b's provider mutex is + # `WorldPool`'s own `_provider_lock` now (mutation-verified: it serializes + # provision/reset/close/healthy under one lock) -- wired directly, no extra wrapper. + provider = deps.build_provider() pool = WorldPool( provider, bundle=manifest, source=source, bundle_dir=bundle_dir, work_directory=work_directory, instances=parallelism, outbound=adapter, @@ -1502,13 +1494,13 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m code="scenario_preallocation_failed", message=str(exc), ) except ProcessRuntimeError as exc: - # §2f's own domain (never the flattened `infrastructure`/"provision_failed" every - # provisioning failure used to get), stage `building_environment` per §2f. + # §2f's own CARRIED domain (never the flattened `infrastructure`/"provision_failed" + # every provisioning failure used to get), stage `building_environment` per §2f. if adapter.is_fenced: await _bounded_close() return EXIT_FENCED return await _fail( - domain=_process_runtime_error_domain(exc, manifest), + domain=_process_runtime_error_domain(exc), fail_stage=HarnessStage.BUILDING_ENVIRONMENT, code=_section_2f_code(exc.code), message=str(exc), ) @@ -1612,6 +1604,19 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m code="scenario_preallocation_failed", message=str(exc), ) + # Defense against a malformed scenario entry (K1) reaching the scheduler, which reads + # `scenario_key`/`sub_goals`/`setup`/`ready` with plain attribute access and would raise + # AttributeError instead of failing the job cleanly. + scenario_defect = _validate_scenarios(scenarios) + if scenario_defect is not None: + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code=_SCENARIO_ENTRY_INVALID_CODE, message=scenario_defect, + ) + # cancel/fence check at the post-pre-allocation stage boundary. if cancel_requested(): return await _canceled() @@ -1696,7 +1701,6 @@ def main(argv: list[str] | None = None) -> int: "ScenarioSource", "ScenarioSourceNotWired", "ScenariosClient", - "SerializingProvider", "WorldFactoryError", "install_sigterm_handler", "job_secret_purposes", diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py index 5d4672e1..c54e92e0 100644 --- a/src/fi/alk/harness/hosted_scheduler.py +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -32,6 +32,7 @@ import asyncio import inspect +import logging import random import re import threading @@ -42,7 +43,13 @@ from .job import FailureDomain, HarnessStage from .outbound import HostedAttemptSupersededError, HostedChannelFailedError, HostedFencedError -from .process_runtime import EnvironmentRuntime, ProcessRuntimeError, RuntimeState +from .process_runtime import ( + SECTION_2F_DOMAIN, + EnvironmentRuntime, + ProcessRuntimeError, + RuntimeState, +) + from .world.errors import ( WorldError, WorldQueryRejected, @@ -54,6 +61,8 @@ ) from .world.runtime import Call +logger = logging.getLogger(__name__) + # --- the World handle (world-handle-interface.md v3.4) -------------------------------------- # # The frozen contract's code block gives six verbs plus `world_index`/`rng`. `read_only()` is not @@ -308,21 +317,33 @@ async def receipt(self, receipt: ResultReceipt) -> None: ... # these used to be discarded at the reset()/provision() seam (caught as a bare `Exception`, only # `str()` surviving into `world_unhealthy.cause`), so a deterministic `environment`/`agent` fault # (never retried) was re-reported as `world_pool_exhausted`/infrastructure and burned every -# whole-job retry on a failure that repeats identically. `spawn_failed` is contractually split -# managed->infrastructure / source->agent, but this module deliberately never reads `bundle` (see -# the module docstring) so it cannot tell which process kind failed at this seam -- conservatively -# `infrastructure` (matches today's behavior on the source-process half; the correct split is an -# open contract question, not resolved here). -_SECTION_2F_DOMAIN: dict[str, FailureDomain] = { - "source_tree_unavailable": FailureDomain.ENVIRONMENT, - "build_failed": FailureDomain.AGENT, - "runtime_unsupported": FailureDomain.ENVIRONMENT, - "spawn_failed": FailureDomain.INFRASTRUCTURE, - "depends_on_timeout": FailureDomain.INFRASTRUCTURE, - "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, - "seed_failed": FailureDomain.ENVIRONMENT, - "store_statement_failed": FailureDomain.INFRASTRUCTURE, -} +# whole-job retry on a failure that repeats identically. v1.15: the PRODUCER now resolves +# `spawn_failed`'s managed-vs-source split at the raise site (`ProcessRuntimeError.domain`) -- +# this module reads that carried domain first; `SECTION_2F_DOMAIN` (imported from +# `process_runtime.py`, the codes' own home) is consulted only for the rare error that reaches +# here with no carried domain, and that fallback is logged so a silent re-guess never hides again. + + +def _resolve_2f_domain( + code: str | None, domain: FailureDomain | None, +) -> tuple[str, FailureDomain] | None: + """v1.15 §2f: pair a code with its resolved domain. `domain` should be the value CARRIED by a + typed provisioner error (`ProcessRuntimeError.domain`); `None` here falls back to the closed + code->domain map (and logs it) rather than silently re-guessing. Returns `None` outright for + no code, or a code outside the §2f table (`internal_*` etc.) — callers already treat that the + same as "not a §2f error." + """ + if code is None or code not in SECTION_2F_DOMAIN: + return None + if domain is not None: + return code, domain + logger.warning( + "process_runtime error %r crossed the §4 seam with no carried domain; using the §2f " + "fallback map (%s)", code, SECTION_2F_DOMAIN[code].value, + ) + return code, SECTION_2F_DOMAIN[code] + + # v1.13: only these two domains are "never retried" -- a uniform §2f code across every unhealthy # world in one of them surfaces as that code+domain; anything else (mixed codes, or any # infrastructure-domain fault) stays `world_pool_exhausted` exactly as before. @@ -655,10 +676,12 @@ def __init__( self._down: set[int] = set() self._fresh: set[int] = set() # m9: provisioned/recovered but never yet leased/reset self._effective_size = 0 # R2: the achieved world count `start()` settled on - # The §2f code (or `None`) behind the most recent demotion/reconcile-failure for a down - # world index -- read by `lease()`'s exhaustion check to decide whether a uniform - # never-retried code can surface instead of the generic `world_pool_exhausted`. - self._down_codes: dict[int, str | None] = {} + # The §2f (code, domain) pair (or `None`) behind the most recent demotion/reconcile-failure + # for a down world index -- read by `lease()`'s exhaustion check to decide whether a + # uniform never-retried code can surface instead of the generic `world_pool_exhausted`. + # `domain` is the CARRIED value off the typed error (v1.15), captured once here rather than + # re-derived later from the code alone. + self._down_codes: dict[int, tuple[str, FailureDomain] | None] = {} self._fenced: BaseException | None = None # latched by mark_fenced(), never cleared # m1: `asyncio.Condition` (not a manual `Event` + `clear()`) — waiting and notifying share @@ -813,9 +836,9 @@ async def lease( codes = {self._down_codes.get(index) for index in self._down} code = domain = None if len(codes) == 1: - (only_code,) = codes - if only_code is not None: - only_domain = _SECTION_2F_DOMAIN.get(only_code) + (only,) = codes + if only is not None: + only_code, only_domain = only if only_domain in _SECTION_2F_NEVER_RETRIED: code, domain = only_code, only_domain raise NoWorldsAvailable( @@ -890,16 +913,15 @@ async def lease( if reset_exc is not None else f"reset left world in state {runtime.state.value}" ) - # Preserve a typed §2f code across this seam instead of flattening it to free - # text -- `mark_unhealthy` records it so a later exhaustion declaration can tell a - # deterministic never-retried fault apart from a generic infrastructure one. - code = ( - reset_exc.code - if isinstance(reset_exc, ProcessRuntimeError) and reset_exc.code in _SECTION_2F_DOMAIN - else None - ) - - await self.mark_unhealthy(world_index, cause=cause, code=code) + # Preserve a typed §2f code (and its CARRIED domain, v1.15) across this seam + # instead of flattening it to free text -- `mark_unhealthy` records it so a later + # exhaustion declaration can tell a deterministic never-retried fault apart from a + # generic infrastructure one. + is_typed = isinstance(reset_exc, ProcessRuntimeError) + code = reset_exc.code if is_typed else None + domain = reset_exc.domain if is_typed else None + + await self.mark_unhealthy(world_index, cause=cause, code=code, domain=domain) # loop again — this index is now excluded via `_down`, no explicit retry bookkeeping. async def release(self, world_index: int) -> None: @@ -909,7 +931,17 @@ async def release(self, world_index: int) -> None: self._available.add(world_index) self._state_lock.notify_all() - async def mark_unhealthy(self, world_index: int, *, cause: str, code: str | None = None) -> None: + async def mark_unhealthy( + self, + world_index: int, + *, + cause: str, + code: str | None = None, + domain: FailureDomain | None = None, + ) -> None: + # `domain` is the CARRIED value off a typed provisioner error (v1.15); `None` here (e.g. a + # caller that only has a bare code) falls back to the closed map via `_resolve_2f_domain`, + # logged when it fires. async with self._state_lock: self._leased.discard(world_index) self._available.discard(world_index) @@ -918,7 +950,7 @@ async def mark_unhealthy(self, world_index: int, *, cause: str, code: str | None # Unconditional -- every demotion overwrites the recorded reason (or clears a stale # §2f code with `None` when this one isn't typed), so exhaustion always reads the # MOST RECENT cause for this index, never a leftover from an earlier failure. - self._down_codes[world_index] = code + self._down_codes[world_index] = _resolve_2f_domain(code, domain) runtime = self._runtimes.get(world_index) if runtime is not None: # M12 (spine v1.12 §4.5b, normative): the scheduler demotes `state` on the @@ -999,14 +1031,13 @@ async def _reconcile(self) -> None: break if last_exc is not None or runtimes is None: - # The FINAL failed re-provision attempt's typed §2f code, applied to every world - # still down when this reconcile gives up -- one `provision()` call covers the whole - # pool, so a typed failure here is uniform by construction across everything it did - # not just recover. - code = ( - last_exc.code - if isinstance(last_exc, ProcessRuntimeError) and last_exc.code in _SECTION_2F_DOMAIN - else None + # The FINAL failed re-provision attempt's typed §2f code (and its CARRIED domain, + # v1.15), applied to every world still down when this reconcile gives up -- one + # `provision()` call covers the whole pool, so a typed failure here is uniform by + # construction across everything it did not just recover. + is_typed = isinstance(last_exc, ProcessRuntimeError) + code_and_domain = _resolve_2f_domain( + last_exc.code if is_typed else None, last_exc.domain if is_typed else None, ) # R8: every success path below ends in `notify_all()` — this give-up path must too, # or a `lease()` blocked in `_wait_bounded(poll=False)` (the `abandon is None` case) @@ -1016,7 +1047,7 @@ async def _reconcile(self) -> None: # exhaustion later reads that leftover code as if it were this attempt's own result. async with self._state_lock: for index in self._down: - self._down_codes[index] = code + self._down_codes[index] = code_and_domain self._state_lock.notify_all() return # stays `_down`; the next `mark_unhealthy` (or a lease-triggered wait) retries. @@ -1036,7 +1067,7 @@ async def _reconcile(self) -> None: # path only fires on a raised/failed `provision()`), so without this the state block below # has no code of its own and would otherwise leave whatever an earlier, superseded demotion # recorded standing. - healthy_codes: dict[int, str | None] = {} + healthy_codes: dict[int, tuple[str, FailureDomain] | None] = {} async with self._provider_lock: for runtime in runtimes: try: @@ -1046,10 +1077,9 @@ async def _reconcile(self) -> None: healthy_codes[runtime.world_index] = None except Exception as exc: # noqa: BLE001 healthy_by_index[runtime.world_index] = False - healthy_codes[runtime.world_index] = ( - exc.code - if isinstance(exc, ProcessRuntimeError) and exc.code in _SECTION_2F_DOMAIN - else None + is_typed = isinstance(exc, ProcessRuntimeError) + healthy_codes[runtime.world_index] = _resolve_2f_domain( + exc.code if is_typed else None, exc.domain if is_typed else None, ) achieved = {runtime.world_index for runtime in runtimes} diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py index ce41aa9f..1fd16078 100644 --- a/src/fi/alk/harness/process_preflight.py +++ b/src/fi/alk/harness/process_preflight.py @@ -127,6 +127,22 @@ def __init__(self, code: str, message: str) -> None: _JOB_SHARED_PORT_BAND = range(14000, 14100) _PER_WORLD_PORT_BAND = range(15000, 15800) +# `process_runtime.py`'s own `_rabbitmq_management_port` formula (`amqp_port + 10000`) — +# mirrored here for the same reason as the two bands above: preflight has no business depending +# on the execution module. The rabbitmq catalog entry supports `datadir_copy` only (no +# `template_database`), so its amqp port is always drawn from the PER-WORLD band in practice +# today; the job-shared shift is reserved too, defensively, since the formula itself is generic +# and nothing about this band's math depends on which base band it is applied to. +_RABBITMQ_MANAGEMENT_PORT_OFFSET = 10000 +_JOB_SHARED_RABBITMQ_MANAGEMENT_BAND = range( + _JOB_SHARED_PORT_BAND.start + _RABBITMQ_MANAGEMENT_PORT_OFFSET, + _JOB_SHARED_PORT_BAND.stop + _RABBITMQ_MANAGEMENT_PORT_OFFSET, +) +_PER_WORLD_RABBITMQ_MANAGEMENT_BAND = range( + _PER_WORLD_PORT_BAND.start + _RABBITMQ_MANAGEMENT_PORT_OFFSET, + _PER_WORLD_PORT_BAND.stop + _RABBITMQ_MANAGEMENT_PORT_OFFSET, +) + def preflight_bundle( bundle_dir: Path, @@ -487,11 +503,17 @@ def _verify_fixed_port_not_reserved(manifest: EnvironmentBundleV2) -> None: for process in manifest.processes: if not isinstance(process, SourceProcess) or process.fixed_port is None: continue - if process.fixed_port in _JOB_SHARED_PORT_BAND or process.fixed_port in _PER_WORLD_PORT_BAND: + if ( + process.fixed_port in _JOB_SHARED_PORT_BAND + or process.fixed_port in _PER_WORLD_PORT_BAND + or process.fixed_port in _JOB_SHARED_RABBITMQ_MANAGEMENT_BAND + or process.fixed_port in _PER_WORLD_RABBITMQ_MANAGEMENT_BAND + ): raise PreflightError( "fixed_port_reserved", f"{process.name}: fixed_port {process.fixed_port} falls inside the provisioner's " - "own port-formula bands (14000-14099 job-shared, 15000-15799 per-world)", + "own port-formula bands (14000-14099 job-shared, 15000-15799 per-world, " + "24000-24099/25000-25799 rabbitmq management)", ) diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py index 092ee005..1272c142 100644 --- a/src/fi/alk/harness/process_runtime.py +++ b/src/fi/alk/harness/process_runtime.py @@ -60,6 +60,7 @@ SourceProcess, StoreEntry, ) +from .job import FailureDomain logger = logging.getLogger(__name__) @@ -80,16 +81,51 @@ class ProcessRuntimeError(RuntimeError): bug to fix here, not a failure the outbound seam ever needs a name for. `stage` names which phase failed (`build`, `spawn`, `depends_on`, `render`); `process` names the process involved, when there is one. + + v1.15 §2f: the PRODUCER resolves `domain` — a row like `spawn_failed` splits managed-vs-source + on a fact (the failing process's declared `kind`) only known at the raise site, so every raise + site for a §2f code sets `domain` here rather than leaving the consumer to re-derive it from + `code` alone (which cannot recover the managed/source split). `domain` is `None` only where + the raise site genuinely cannot tell which process kind failed (a mixed-kind catch-all); such + sites are rare and `SECTION_2F_DOMAIN` below exists as their fallback. """ - def __init__(self, stage: str, code: str, message: str, *, process: str | None = None) -> None: + def __init__( + self, + stage: str, + code: str, + message: str, + *, + process: str | None = None, + domain: FailureDomain | None = None, + ) -> None: self.stage = stage self.code = code self.process = process + self.domain = domain located = f" ({process})" if process else "" super().__init__(f"{stage}/{code}{located}: {message}") +# §2f's closed build/run failure-code table (hosted-execution-seams.md §4.6) -> FailureDomain. +# THE single home for this map (v1.15's §2f-map cleanup) — `hosted_scheduler.py`/ +# `hosted_entrypoint.py` import it rather than keeping their own copies. It is a FALLBACK ONLY: +# every raise site above sets `ProcessRuntimeError.domain` directly, so a consumer reads that +# first and falls back to this map only for an error that reaches it with no carried domain. +# `spawn_failed`'s entry is the pre-v1.15 conservative guess (matches its source-process half); +# a `spawn_failed` raised with a resolved `domain` never consults this entry at all. +SECTION_2F_DOMAIN: dict[str, FailureDomain] = { + "source_tree_unavailable": FailureDomain.ENVIRONMENT, + "build_failed": FailureDomain.AGENT, + "runtime_unsupported": FailureDomain.ENVIRONMENT, + "spawn_failed": FailureDomain.INFRASTRUCTURE, + "depends_on_timeout": FailureDomain.INFRASTRUCTURE, + "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, + "seed_failed": FailureDomain.ENVIRONMENT, + "store_statement_failed": FailureDomain.INFRASTRUCTURE, +} + + # --- §3 EnvironmentRuntime ------------------------------------------------------------------- @@ -275,6 +311,7 @@ def render_capability_address( raise ProcessRuntimeError( "render", "unsupported_capability_protocol", f"{protocol.value} has no defined address shape at this seam", + domain=FailureDomain.ENVIRONMENT, ) @@ -493,6 +530,7 @@ def _resolve_process_user( require: bool, process_name: str, stage: str, + domain: FailureDomain, ) -> "pwd.struct_passwd | None": """F1, p5-round1-review: every build tree and every spawned process must run under its declared `user`, not the harness's own `svc-control` — otherwise an untrusted `agent` process @@ -503,7 +541,9 @@ def _resolve_process_user( `require=False` (the default, the local test lane's shape — no `svc-*` accounts on a dev box) logs and returns `None`, so the caller runs unprivileged rather than failing every local run. `require=True` is for a caller that knows it is on the hosted path, where the snapshot's own - guarantee means resolution failing is itself an infrastructure fault worth a typed failure. + guarantee means resolution failing is itself a typed failure. `domain` is the caller's own + §2f `spawn_failed` split (managed vs source) — this function has no process-kind knowledge of + its own, so it never guesses it. """ resolved = resolver(user.value) if resolved is None: @@ -511,7 +551,7 @@ def _resolve_process_user( raise ProcessRuntimeError( stage, "spawn_failed", f"{user.value!r} has no passwd entry; the hosted snapshot must guarantee it", - process=process_name, + process=process_name, domain=domain, ) logger.warning( "process %s declares user=%s but it is not resolvable on this host; running " @@ -576,7 +616,7 @@ def _reject_escaping_symlinks(tree_root: Path, allowed_root: Path, *, process_na "build", "source_tree_unavailable", f"{entry.relative_to(tree_root)} is a symlink to {target}, which escapes " "/work/source", - process=process_name, + process=process_name, domain=FailureDomain.ENVIRONMENT, ) @@ -627,7 +667,7 @@ def build_process_tree( raise ProcessRuntimeError( "build", "source_tree_unavailable", f"{process.working_directory} resolves outside /work/source", - process=process.name, + process=process.name, domain=FailureDomain.ENVIRONMENT, ) if not resolved_source_dir.is_dir(): # F5, p5-round1-review: named explicitly, before any copy attempt, rather than letting @@ -635,7 +675,7 @@ def build_process_tree( raise ProcessRuntimeError( "build", "source_tree_unavailable", f"{process.working_directory} is absent or not a directory in the checkout", - process=process.name, + process=process.name, domain=FailureDomain.ENVIRONMENT, ) _reject_escaping_symlinks(resolved_source_dir, resolved_source_root, process_name=process.name) @@ -650,11 +690,12 @@ def build_process_tree( raise ProcessRuntimeError( "build", "source_tree_unavailable", f"copying {process.working_directory}: {exc}", process=process.name, + domain=FailureDomain.ENVIRONMENT, ) from exc resolved_user = _resolve_process_user( process.user, resolver=user_resolver, require=require_declared_user, - process_name=process.name, stage="build", + process_name=process.name, stage="build", domain=FailureDomain.AGENT, ) if resolved_user is not None: _chown_tree(build_dir, uid=resolved_user.pw_uid, gid=resolved_user.pw_gid, chown=chown) @@ -676,7 +717,7 @@ def build_process_tree( raise ProcessRuntimeError( "build", "build_failed", f"{step!r} exceeded the {build_step_timeout_seconds}s build-step timeout", - process=process.name, + process=process.name, domain=FailureDomain.AGENT, ) from exc except FileNotFoundError as exc: if not build_dir.is_dir(): @@ -687,6 +728,7 @@ def build_process_tree( raise ProcessRuntimeError( "build", "source_tree_unavailable", f"{build_dir} vanished before {step!r} could run", process=process.name, + domain=FailureDomain.ENVIRONMENT, ) from exc if _looks_like_missing_interpreter(step[0]): raise ProcessRuntimeError( @@ -694,10 +736,11 @@ def build_process_tree( "runtime_unsupported", f"{step[0]!r} is not on the snapshot's PATH; the snapshot ships python " "3.11/3.12 and node 20/22 only", - process=process.name, + process=process.name, domain=FailureDomain.ENVIRONMENT, ) from exc raise ProcessRuntimeError( - "build", "build_failed", f"{step!r}: {exc}", process=process.name + "build", "build_failed", f"{step!r}: {exc}", process=process.name, + domain=FailureDomain.AGENT, ) from exc if result.returncode != 0: stderr = (result.stderr or "").strip()[:2000] @@ -705,7 +748,7 @@ def build_process_tree( "build", "build_failed", f"{step!r} exited {result.returncode}" + (f": {stderr}" if stderr else ""), - process=process.name, + process=process.name, domain=FailureDomain.AGENT, ) return build_dir @@ -1028,7 +1071,7 @@ def spawn_managed_process( """ resolved_user = _resolve_process_user( process.user, resolver=user_resolver, require=require_declared_user, - process_name=process.name, stage="spawn", + process_name=process.name, stage="spawn", domain=FailureDomain.INFRASTRUCTURE, ) try: data_dir.mkdir(parents=True, exist_ok=True) @@ -1056,7 +1099,7 @@ def spawn_managed_process( # seams themselves. raise ProcessRuntimeError( "spawn", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) from exc spawn_uid = resolved_user.pw_uid if resolved_user is not None else None spawn_gid = resolved_user.pw_gid if resolved_user is not None else None @@ -1066,7 +1109,7 @@ def spawn_managed_process( if credentials is None: raise ProcessRuntimeError( "spawn", "spawn_failed", "postgres requires generated credentials", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) if not (data_dir / "PG_VERSION").exists(): pwfile = data_dir.parent / f".{process.name}.pwfile" @@ -1096,7 +1139,7 @@ def spawn_managed_process( raise ProcessRuntimeError( "spawn", "spawn_failed", f"initdb exited {result.returncode}: {stderr}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) finally: pwfile.unlink(missing_ok=True) @@ -1107,7 +1150,7 @@ def spawn_managed_process( if credentials is None: raise ProcessRuntimeError( "spawn", "spawn_failed", "rabbitmq requires generated credentials", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) # M8, p6-review-r1: written fresh on every spawn (job bootstrap AND every world's own # instance) — cheap, and means a `datadir_copy` restore can never carry a stale plugin/ @@ -1134,7 +1177,7 @@ def spawn_managed_process( except OSError as exc: raise ProcessRuntimeError( "spawn", "spawn_failed", f"{process.name}: writing rabbitmq config: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) from exc env.update(rabbitmq_daemon_env(data_dir=data_dir, port=port, credentials=credentials)) env["RABBITMQ_ENABLED_PLUGINS_FILE"] = str(plugins_path) @@ -1149,7 +1192,8 @@ def spawn_managed_process( argv = rabbitmq_daemon_argv() else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. raise ProcessRuntimeError( - "spawn", "spawn_failed", f"unknown engine {process.engine!r}", process=process.name + "spawn", "spawn_failed", f"unknown engine {process.engine!r}", process=process.name, + domain=FailureDomain.INFRASTRUCTURE, ) try: handle = runner( @@ -1157,7 +1201,10 @@ def spawn_managed_process( user=spawn_uid, group=spawn_gid, ) except FileNotFoundError as exc: - raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc + raise ProcessRuntimeError( + "spawn", "spawn_failed", str(exc), process=process.name, + domain=FailureDomain.INFRASTRUCTURE, + ) from exc return SpawnedWorldProcess( process_name=process.name, handle=handle, port=port, world_index=None, uid=spawn_uid, gid=spawn_gid, @@ -1188,7 +1235,7 @@ def spawn_source_process( """ resolved_user = _resolve_process_user( process.user, resolver=user_resolver, require=require_declared_user, - process_name=process.name, stage="spawn", + process_name=process.name, stage="spawn", domain=FailureDomain.AGENT, ) try: world_dir.mkdir(parents=True, exist_ok=True) @@ -1199,7 +1246,7 @@ def spawn_source_process( # a permission error creating/chowning this process's `{{WORLD_DIR}}` used to raise bare. raise ProcessRuntimeError( "spawn", "spawn_failed", f"{process.name}: preparing {world_dir}: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.AGENT, ) from exc rendered = render_environment( process, @@ -1221,7 +1268,9 @@ def spawn_source_process( group=resolved_user.pw_gid if resolved_user is not None else None, ) except FileNotFoundError as exc: - raise ProcessRuntimeError("spawn", "spawn_failed", str(exc), process=process.name) from exc + raise ProcessRuntimeError( + "spawn", "spawn_failed", str(exc), process=process.name, domain=FailureDomain.AGENT, + ) from exc port = port_plan.port_for(process.name, world_index) return SpawnedWorldProcess( process_name=process.name, handle=handle, port=port, world_index=world_index, @@ -1449,7 +1498,7 @@ def probe_ready() -> bool: "depends_on", "depends_on_timeout", f"{dependency_name}: readiness probe did not pass within {combined_timeout}s", - process=dependency_name, + process=dependency_name, domain=FailureDomain.INFRASTRUCTURE, ), ) return @@ -1486,7 +1535,7 @@ def condition() -> bool: "depends_on_timeout", f"{dependency_name}: started_check did not pass within " f"{started_check.timeout_seconds}s", - process=dependency_name, + process=dependency_name, domain=FailureDomain.INFRASTRUCTURE, ), ) @@ -1622,7 +1671,7 @@ def _call_sql( raise ProcessRuntimeError( stage, "store_statement_failed", f"{process_name}: store rejected a provisioner-issued statement: {exc}", - process=process_name, + process=process_name, domain=FailureDomain.INFRASTRUCTURE, ) from exc @@ -1636,7 +1685,7 @@ def _call_redis( raise ProcessRuntimeError( stage, "store_statement_failed", f"{process_name}: store rejected a provisioner-issued command: {exc}", - process=process_name, + process=process_name, domain=FailureDomain.INFRASTRUCTURE, ) from exc @@ -1650,7 +1699,7 @@ def _call_rabbitmq( raise ProcessRuntimeError( stage, "store_statement_failed", f"{process_name}: store rejected a provisioner-issued queue inspection: {exc}", - process=process_name, + process=process_name, domain=FailureDomain.INFRASTRUCTURE, ) from exc @@ -1699,7 +1748,7 @@ def _call_rabbitmq_action( raise ProcessRuntimeError( stage, "store_statement_failed", f"{process_name}: store rejected the provisioner's canary {action}: {exc}", - process=process_name, + process=process_name, domain=FailureDomain.INFRASTRUCTURE, ) from exc @@ -1910,18 +1959,20 @@ def apply_seed_file( except Exception as exc: raise ProcessRuntimeError( "seed", "seed_failed", f"{file}: {exc}", process=process_name, + domain=FailureDomain.ENVIRONMENT, ) from exc return else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. raise ProcessRuntimeError( - "seed", "seed_failed", f"unknown engine {engine!r}", process=process_name + "seed", "seed_failed", f"unknown engine {engine!r}", process=process_name, + domain=FailureDomain.ENVIRONMENT, ) if result.returncode != 0: stderr = (result.stderr or "").strip()[:2000] raise ProcessRuntimeError( "seed", "seed_failed", f"{file}: exited {result.returncode}" + (f": {stderr}" if stderr else ""), - process=process_name, + process=process_name, domain=FailureDomain.ENVIRONMENT, ) @@ -2393,6 +2444,7 @@ def ready() -> bool: timeout_error=lambda: ProcessRuntimeError( "baseline", "depends_on_timeout", f"{process.name}: did not become ready within {timeout}s", process=process.name, + domain=FailureDomain.INFRASTRUCTURE, ), ) @@ -2638,7 +2690,7 @@ def _freeze_one_store( raise ProcessRuntimeError( "baseline", "seed_failed", f"{process.name}: sentinel check failed against the freshly seeded baseline", - process=process.name, + process=process.name, domain=FailureDomain.ENVIRONMENT, ) row_counts: dict[str, int] = {} @@ -2702,7 +2754,7 @@ def _freeze_one_store( raise ProcessRuntimeError( "baseline", "store_statement_failed", f"{process.name}: sealing the datadir_copy baseline: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) from exc baseline_reference = str(baseline_dir) # `result_handle` stays `None` — every world starts its OWN engine instance from this @@ -2793,7 +2845,7 @@ def _seal_world_store( raise ProcessRuntimeError( "reset", "store_statement_failed", f"{process.name}: restoring the datadir_copy baseline: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) from exc new_handle = spawn_managed_process( process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, @@ -2835,7 +2887,7 @@ def _seal_world_store( # class as `spawn_managed_process`'s own boundary. raise ProcessRuntimeError( "reset", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", - process=process.name, + process=process.name, domain=FailureDomain.INFRASTRUCTURE, ) from exc handle = spawn_managed_process( process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, @@ -3247,15 +3299,13 @@ def run_conformance_gate( def _read_job_secret_purposes(work_directory: Path) -> dict[str, str]: """§0.2/§4.1: `/work/job.json` is the provisioner's own configuration source — `agent. secret_refs` (§1: `{alias: {manager, key, version, purpose}}`) is where each alias's REAL - purpose comes from. N10, p6-review-r2 (MAJOR): every alias used to be relabelled `target_ - provider` unconditionally in `_load_and_delete_secrets`, which silently defeats `select_ - process_secrets`'s own `SOURCE_CHECKOUT` exclusion (F13) the moment a `source_checkout` alias - ever reaches `secrets.json` — the guest must not depend on the gateway alone never putting one - there, which is exactly the promise F13's own docstring says it will not depend on. Q11, - p6-review-r3: the two raises below use `spawn_failed` for a malformed `job.json`, which is - also a vocabulary stretch — §2f's domain rule for it is "infrastructure if a managed engine, - `agent` if source," neither of which is what a config-read fault actually is; picked as the - closest §4.6 code available, same reasoning as `store_statement_failed`'s stretch elsewhere. + purpose comes from — never relabelled `target_provider` unconditionally, which would silently + defeat `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion the moment a + `source_checkout` alias ever reaches `secrets.json`. The two raises below use `spawn_failed` + for a malformed `job.json`, which is a vocabulary stretch — §2f's domain rule for it is + "infrastructure if a managed engine, `agent` if source," neither of which is what a + config-read fault actually is; picked as the closest §4.6 code available, domain + `infrastructure` (a corrupted upload, not a bundle-authoring fault the customer caused). """ job_json_path = work_directory / "job.json" if not job_json_path.exists(): @@ -3275,11 +3325,13 @@ def _read_job_secret_purposes(work_directory: Path) -> dict[str, str]: except (OSError, json.JSONDecodeError) as exc: raise ProcessRuntimeError( "secrets", "spawn_failed", f"{job_json_path}: unreadable or not valid JSON: {exc}", + domain=FailureDomain.INFRASTRUCTURE, ) from exc if not isinstance(raw, dict): raise ProcessRuntimeError( "secrets", "spawn_failed", f"{job_json_path}: expected a JSON object, got {type(raw).__name__}", + domain=FailureDomain.INFRASTRUCTURE, ) agent = raw.get("agent") refs = agent.get("secret_refs") if isinstance(agent, dict) else None @@ -3300,10 +3352,10 @@ def _load_and_delete_secrets( in-memory map lives for the whole job — `reset` restarts and `provision` reconciliations re-inject from memory." - N10, p6-review-r2 (MAJOR): each alias's purpose comes from `job.json`'s own `agent. - secret_refs` (`_read_job_secret_purposes`) — NOT invented as `target_provider` for every - alias, which used to retire `select_process_secrets`'s `SOURCE_CHECKOUT` exclusion the moment - a `source_checkout` alias ever reached this file. `secret_purpose_map`, when given, overrides + Each alias's purpose comes from `job.json`'s own `agent.secret_refs` + (`_read_job_secret_purposes`) — NOT invented as `target_provider` for every alias, which would + retire `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion the moment a + `source_checkout` alias ever reached this file. `secret_purpose_map`, when given, overrides the job.json read entirely (the local/test lane's own shape — no `/work/job.json` on a dev box). An alias in `secrets.json` with no matching ref anywhere is dropped, not injected under a guessed purpose, and logged — `select_process_secrets` naturally never matches an alias @@ -3317,6 +3369,7 @@ def _load_and_delete_secrets( except (OSError, json.JSONDecodeError) as exc: raise ProcessRuntimeError( "secrets", "spawn_failed", f"{secrets_path}: unreadable or not valid JSON: {exc}", + domain=FailureDomain.INFRASTRUCTURE, ) from exc secrets_path.unlink(missing_ok=True) if not isinstance(raw, dict): @@ -3324,6 +3377,7 @@ def _load_and_delete_secrets( "secrets", "spawn_failed", f"{secrets_path}: expected a JSON object of alias -> value, got " f"{type(raw).__name__}", + domain=FailureDomain.INFRASTRUCTURE, ) values = {str(alias): str(value) for alias, value in raw.items()} @@ -3517,15 +3571,19 @@ def _provision_sync( bundle, bundle_digest=bundle_digest, context=context ) except (OSError, shutil.Error) as exc: - # N9, p6-review-r2 (MAJOR): §4.6 — filesystem failures during provisioning are - # `infrastructure`. This phase's own copy-heavy work (build trees, baseline - # snapshot/seal) used to be able to raise bare here. Q11, p6-review-r3: - # `store_statement_failed` is a vocabulary stretch for a build-tree copy fault — - # the closest §2f code in a closed table with no generic `provisioner_io_failed`, - # and it lands the correct `infrastructure` domain either way. + # §4.6 — filesystem failures during provisioning are `infrastructure`. This + # phase's own copy-heavy work (build trees, baseline snapshot/seal) can raise a raw + # filesystem error covering EITHER a source build tree or a managed baseline seal — + # `store_statement_failed` is a vocabulary stretch for a build-tree copy fault (the + # closest §2f code in a closed table with no generic `provisioner_io_failed`), but + # the domain is `infrastructure` either way, so it is set directly here rather than + # left to the fallback map. self._manifest = None self._bundle_digest = None - raise ProcessRuntimeError("baseline", "store_statement_failed", str(exc)) from exc + raise ProcessRuntimeError( + "baseline", "store_statement_failed", str(exc), + domain=FailureDomain.INFRASTRUCTURE, + ) from exc except BaseException: self._manifest = None self._bundle_digest = None @@ -3654,15 +3712,16 @@ def _ensure_world(self, world_index: int) -> None: existing_handles=self._world_handles.get(world_index, {}), ) except (OSError, shutil.Error) as exc: - # N9, p6-review-r2 (MAJOR): §4.6 — filesystem failures during provisioning are - # infrastructure. mkdir/chown/chmod for a (re)spawned process, uncaught this deep, - # used to raise bare out of `provision()`. + # A raw filesystem fault this deep in `_clone_or_reset_world` (mkdir/chown/chmod for a + # (re)spawned process) does not by itself say which process's kind failed — `domain` + # is left unset so the §4 seam's fallback map applies (SECTION_2F_DOMAIN), since every + # raise site that DOES know its process kind already carries its own domain directly. partial = getattr(exc, "partial_handles", None) if partial is not None: - self._world_handles[world_index] = partial # N4: never orphan a live engine. + self._world_handles[world_index] = partial # never orphan a live engine. raise ProcessRuntimeError("spawn", "spawn_failed", str(exc)) from exc except BaseException as exc: - # N4, p6-review-r2 (MAJOR): a raise partway through `_clone_or_reset_world`/`spawn_ + # A raise partway through `_clone_or_reset_world`/`spawn_ # world` used to drop every ALREADY-(re)sealed/spawned handle of THIS world on the # floor — nothing held it, so `close()`'s own `rmtree` of this world's data directory # next ran against a still-live server. `exc.partial_handles` (set by `freeze_ @@ -3782,12 +3841,14 @@ def _reset_sync(self, runtime: EnvironmentRuntime) -> None: existing_handles=self._world_handles.get(world_index, {}), ) except (OSError, shutil.Error) as exc: - # N9, p6-review-r2 (MAJOR): reset's own filesystem work is fundamentally "reseal this - # world's stores from baseline" — the `store_statement_failed` half of N9's mapping. + # reset's own filesystem work is fundamentally "reseal this world's stores from + # baseline" — the infrastructure-domain half of the store-statement mapping. partial = getattr(exc, "partial_handles", None) if partial is not None: - self._world_handles[world_index] = partial # N4. - raise ProcessRuntimeError("reset", "store_statement_failed", str(exc)) from exc + self._world_handles[world_index] = partial + raise ProcessRuntimeError( + "reset", "store_statement_failed", str(exc), domain=FailureDomain.INFRASTRUCTURE, + ) from exc except BaseException as exc: partial = getattr(exc, "partial_handles", None) # N4, p6-review-r2. if partial is not None: @@ -3981,7 +4042,7 @@ def _respawn_dead_job_shared_engine( raise ProcessRuntimeError( "reset", "store_statement_failed", f"{process_name}: job-shared engine died and could not be respawned: {exc}", - process=process_name, + process=process_name, domain=FailureDomain.INFRASTRUCTURE, ) from exc return new_handle diff --git a/src/fi/alk/harness/world/stores/container.py b/src/fi/alk/harness/world/stores/container.py index 7544ab58..3cbe08e1 100644 --- a/src/fi/alk/harness/world/stores/container.py +++ b/src/fi/alk/harness/world/stores/container.py @@ -169,8 +169,13 @@ def _await_ready(self) -> None: last = exc time.sleep(0.25) logs = docker("logs", "--tail", "20", self.container, check=False) + container = self.container + # A container that never answers is not a container to leave running -- `start()` already + # set `_started`, so without this the caller's own teardown never runs (nothing ever calls + # `stop()` on a store whose `start()` raised) and the container leaks for good. + self.stop() raise StoreError( - f"{self.container} did not answer within {READY_TIMEOUT_SECONDS:.0f}s: {last}\n" + f"{container} did not answer within {READY_TIMEOUT_SECONDS:.0f}s: {last}\n" f"last lines of its log:\n{logs}" ) diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 3a553baf..19a6e7db 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -9,6 +9,7 @@ from __future__ import annotations import asyncio +import contextlib import hashlib import json import os @@ -38,11 +39,13 @@ ReceiptFailure, ResultReceipt, RunResult, + WorldPool, ) from fi.alk.harness.job import ( AgentConnection, ArtifactLevel, ExecutionMode, + FailureDomain, HarnessArtifactPolicy, HarnessJob, HarnessStage, @@ -280,7 +283,17 @@ def terminal_events(self) -> list[dict[str, Any]]: class FakeProvisioner: - name = "fake-process" # mirrors `ProcessRuntimeProvider.name` so passthrough is testable. + """`_serialized` records any overlapping call into `overlaps` rather than asserting in-band -- + an in-band `assert not self._busy` fires INSIDE `WorldPool.lease()`'s own + `except Exception as exc: reset_exc = exc` (a reset/healthy failure is expected there) or + `_reconcile`'s `except Exception` (a reconcile must never crash the pool), so production + swallows it and a caller driving this fake through `WorldPool` (rather than calling it + directly) would never see the failure. Asserting on `overlaps` from the test's OWN frame is + what actually makes a `_provider_lock` regression observable end-to-end (R7: this is the same + fake `WorldPool`'s own test suite already exercises this way, now needed here too since + `hosted_entrypoint.py` no longer wraps the provider in its own lock).""" + + name = "fake-process" # mirrors `ProcessRuntimeProvider.name`. def __init__(self, instances: int = 1, *, always_unhealthy: bool = False) -> None: self.instances = instances @@ -289,7 +302,8 @@ def __init__(self, instances: int = 1, *, always_unhealthy: bool = False) -> Non self.reset_calls = 0 self.healthy_calls = 0 self.closed = False - self._busy = False + self.overlaps: list[str] = [] + self._in_flight: list[str] = [] self._runtimes = { i: EnvironmentRuntime( runtime_id=f"digest:w{i}", world_index=i, bundle_digest="digest", @@ -298,42 +312,43 @@ def __init__(self, instances: int = 1, *, always_unhealthy: bool = False) -> Non for i in range(instances) } - async def _serialized(self) -> None: - assert not self._busy, "provider called reentrantly" - self._busy = True + @contextlib.asynccontextmanager + async def _serialized(self, label: str): + if self._in_flight: + self.overlaps.append(f"{self._in_flight[-1]} overlapped {label}") + self._in_flight.append(label) try: await asyncio.sleep(0) + yield finally: - self._busy = False + self._in_flight.remove(label) async def provision( self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, contract: Any | None = None, instances: int = 1, ) -> list[EnvironmentRuntime]: del bundle, source, bundle_dir, work_directory, contract - await self._serialized() - self.provision_calls += 1 - return [self._runtimes[i] for i in range(instances)] + async with self._serialized("provision"): + self.provision_calls += 1 + return [self._runtimes[i] for i in range(instances)] async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: del work_directory - await self._serialized() - self.reset_calls += 1 - runtime.state = RuntimeState.READY + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + runtime.state = RuntimeState.READY async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: - del runtime, work_directory - # v1.12 folds `healthy` into the same non-reentrant set as provision/reset/close -- - # this is the one verb that previously did NOT call `_serialized()`, so a `SerializingProvider` - # gap here would pass silently without it. - await self._serialized() - self.healthy_calls += 1 - return not self.always_unhealthy + del work_directory + # v1.12 folds `healthy` into the same non-reentrant set as provision/reset/close. + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return not self.always_unhealthy async def close(self, *, work_directory: Path) -> None: del work_directory - await self._serialized() - self.closed = True + async with self._serialized("close"): + self.closed = True class FakeWorld: @@ -661,52 +676,31 @@ def test_cancel_state_reads_reason_from_file() -> None: assert state.reason() is ob.TerminalReason.TTL_EXCEEDED -def test_serializing_provider_serializes_concurrent_provision_calls() -> None: - async def scenario() -> None: - fake = FakeProvisioner(instances=1) - wrapped = he.SerializingProvider(fake) - work = Path(tempfile.mkdtemp(prefix="p10-serial-")) - results = await asyncio.gather( - wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), - wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), - ) - # The real, load-bearing check is INSIDE `FakeProvisioner._serialized()` (an `assert not - # self._busy` around a real `await` yield point) -- if `SerializingProvider` let both calls - # run concurrently, that assertion would raise and this whole coroutine would fail instead - # of returning cleanly. `provision_calls == 2` only confirms both eventually ran. - assert fake.provision_calls == 2 - assert all(len(r) == 1 for r in results) - - asyncio.run(scenario()) - - -def test_serializing_provider_serializes_healthy_against_provision() -> None: - # v1.12 folds `healthy` into the SAME non-reentrant set as provision/reset/close -- - # `FakeProvisioner.healthy` is the one verb that previously did not call `_serialized()` - # (see its own definition above), so this is the only test that would have caught a - # `SerializingProvider` that forgot to wrap `healthy()` in its lock. +def test_world_pool_serializes_concurrent_provider_calls_end_to_end() -> None: + # R7: `hosted_entrypoint.py` used to wrap every provider in `SerializingProvider` before + # `WorldPool` ever saw it -- removed now that `WorldPool`'s own `_provider_lock` covers + # provision/reset/healthy/close (mutation-verified in review: `healthy` rides the same + # non-reentrancy rule as the other three, per v1.12 §4.5b). This repoints the old + # wrapper-level test at the SAME guarantee, one level up: `pool.start()` (a `provision()` call) + # racing `pool._reconcile()` (a `provision()` THEN a `healthy()` call) against the bare, + # unwrapped `FakeProvisioner` this module now wires directly. async def scenario() -> None: fake = FakeProvisioner(instances=1) - wrapped = he.SerializingProvider(fake) - work = Path(tempfile.mkdtemp(prefix="p10-serial-healthy-")) - runtime = fake._runtimes[0] - await asyncio.gather( - wrapped.provision(None, source=work, bundle_dir=work, work_directory=work, instances=1), - wrapped.healthy(runtime, work_directory=work), + work = Path(tempfile.mkdtemp(prefix="p10-pool-serial-")) + pool = WorldPool( + fake, bundle=None, source=work, bundle_dir=work, work_directory=work, instances=1, ) - assert fake.provision_calls == 1 - assert fake.healthy_calls == 1 + await asyncio.gather(pool.start(), pool._reconcile()) + # `overlaps` is populated OUT-OF-BAND by `FakeProvisioner._serialized()` -- an in-band + # `assert` there would fire inside `_reconcile`'s own `except Exception` (a reconcile must + # never crash the pool) and never reach this frame, silently passing a broken lock. + assert fake.overlaps == [] + assert fake.provision_calls >= 1 + await pool.close() asyncio.run(scenario()) -def test_serializing_provider_name_passes_through() -> None: - # §4's `RuntimeProvider` Protocol declares `name: str` -- the wrapper must not hide it. - fake = FakeProvisioner(instances=1) - wrapped = he.SerializingProvider(fake) - assert wrapped.name == "fake-process" - - def test_scenarios_client_provision_unwraps_the_result_envelope() -> None: capabilities = _capabilities() transport = FakeTransport() @@ -1273,18 +1267,19 @@ async def build(self, job, bundle, scenarios_client, *, pool, world_factory): asyncio.run(scenario()) -def test_process_runtime_error_maps_to_the_closed_2f_domain_table() -> None: - # deleting the whole `except ProcessRuntimeError` clause passed 29/29 because - # `ProcessRuntimeError` never appeared anywhere in the suite -- the §2f domain map - # was unexecuted. Drives four real codes through `pool.start()` and checks each domain. - async def run_case(code: str, process: str | None, expected_domain: str) -> None: +def test_process_runtime_error_uses_the_carried_domain_over_the_fallback_map() -> None: + # v1.15 §2f: the producer resolves and carries `domain` at the raise site -- `spawn_failed`'s + # managed/source split is no longer re-derived from a manifest lookup here (mutation: force + # `_process_runtime_error_domain` back to ignoring `exc.domain` and this test catches it, since + # the SAME code with two different carried domains would then collapse to one fallback value). + async def run_case(code: str, domain: FailureDomain | None, expected_domain: str) -> None: class RaisingProvisioner(FakeProvisioner): async def provision( self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, contract: Any | None = None, instances: int = 1, ) -> list[EnvironmentRuntime]: del bundle, source, bundle_dir, work_directory, contract, instances - raise ProcessRuntimeError("build", code, "synthetic failure", process=process) + raise ProcessRuntimeError("build", code, "synthetic failure", domain=domain) harness = _build_harness(scenarios=[], instances=1) harness.deps.build_provider = lambda: RaisingProvisioner(instances=1) @@ -1296,10 +1291,62 @@ async def provision( assert failure["domain"] == expected_domain assert failure["stage"] == "building_environment" + # The same code, two different CARRIED domains -- proves the domain is read off the + # exception, not re-derived from `code`/`process` (the map alone could never distinguish + # these two cases, since both are `spawn_failed`). + asyncio.run(run_case("spawn_failed", FailureDomain.AGENT, "agent")) # source, carried + asyncio.run(run_case("spawn_failed", FailureDomain.INFRASTRUCTURE, "infrastructure")) # managed, carried + # No carried domain (as a raise site outside this module's control might produce) -- the + # closed map is consulted as a fallback only. asyncio.run(run_case("build_failed", None, "agent")) asyncio.run(run_case("seed_failed", None, "environment")) - asyncio.run(run_case("spawn_failed", "postgres", "infrastructure")) # managed process - asyncio.run(run_case("spawn_failed", "agent", "agent")) # source process + + +def test_scenario_entry_missing_scenario_key_fails_cleanly_never_an_attributeerror() -> None: + # karthik-integration-changes.md K1: the Scenario Generation Contract's own model may not + # carry `scenario_key` yet, and `hosted_scheduler.py` reads `scenario.scenario_key` with plain + # attribute access -- an entry that lacks the field entirely used to raise AttributeError deep + # in the scheduler (no terminal event, a crash exit code) instead of failing the job cleanly. + class ScenarioMissingKey: + scenario_id = "id-x" + sub_goals: list[Any] = [] + + def setup(self, world: Any) -> object: + del world + return None + + def ready(self, world: Any) -> object: + del world + return None + + class RawScenarioSource: + """Returns entries verbatim -- unlike `FakeScenarioSource`, never reads `.scenario_key` + itself before handing them back, so the entrypoint's own validation is what is under + test, not this fixture crashing first.""" + + def __init__(self, scenarios: list[Any]) -> None: + self._scenarios = scenarios + + async def build( + self, job: Any, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, + world_factory: Any, + ) -> list[Any]: + del job, bundle, scenarios_client, pool, world_factory + return self._scenarios + + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = RawScenarioSource([ScenarioMissingKey()]) + result = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert result == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "environment" + assert "scenario_key" in failure["message"] + + asyncio.run(scenario()) def test_finish_emits_the_terminal_before_closing_the_pool() -> None: @@ -2125,9 +2172,7 @@ async def slow_emit(self: HostedScheduler, result: RunResult) -> None: test_row_counts_for_capability_returns_the_matching_store, test_row_counts_for_capability_raises_when_the_capability_is_absent, test_cancel_state_reads_reason_from_file, - test_serializing_provider_serializes_concurrent_provision_calls, - test_serializing_provider_serializes_healthy_against_provision, - test_serializing_provider_name_passes_through, + test_world_pool_serializes_concurrent_provider_calls_end_to_end, test_scenarios_client_provision_unwraps_the_result_envelope, test_scenarios_client_fencing_latches_the_shared_channel_state, test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event, @@ -2149,7 +2194,7 @@ async def slow_emit(self: HostedScheduler, result: RunResult) -> None: test_build_json_fixed_port_at_w1_does_not_crash, test_e2e_two_scenarios_one_pass_one_fail_reaches_completed_and_exits_0, test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped, - test_process_runtime_error_maps_to_the_closed_2f_domain_table, + test_process_runtime_error_uses_the_carried_domain_over_the_fallback_map, test_finish_emits_the_terminal_before_closing_the_pool, test_redaction_end_to_end_secret_never_crosses_any_channel, test_drain_loops_past_a_backlog_larger_than_one_batch_and_still_delivers_the_terminal, diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py index a21f1fa4..5208a9a3 100644 --- a/tests/harness/test_hosted_scheduler.py +++ b/tests/harness/test_hosted_scheduler.py @@ -998,6 +998,47 @@ async def provision(self, bundle, *, source, bundle_dir, work_directory, contrac asyncio.run(scenario()) +def test_pool_exhaustion_uses_the_carried_domain_not_the_fallback_map() -> None: + # v1.15 §2f: the producer resolves `spawn_failed`'s managed/source split AT THE RAISE and + # carries it on `ProcessRuntimeError.domain` -- the scheduler must read that directly. The + # fallback map's own `spawn_failed` entry is `infrastructure` (retryable, never surfaced by + # exhaustion); this raises `spawn_failed` carrying `agent` instead, which IS a never-retried + # domain. Only reading the CARRIED domain makes this surface as `spawn_failed`/`agent` -- + # a scheduler that fell back to the map (or ignored `domain` entirely) would see + # `infrastructure`, which is not in `_SECTION_2F_NEVER_RETRIED`, and this would incorrectly + # stay the generic `world_pool_exhausted` instead. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError( + "reset", "spawn_failed", "agent process exec failed", + domain=hs.FailureDomain.AGENT, + ) + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError( + "reset", "spawn_failed", "agent process exec failed", + domain=hs.FailureDomain.AGENT, + ) + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "spawn_failed" + assert exc.domain is hs.FailureDomain.AGENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + def test_reconcile_give_up_with_an_untyped_final_attempt_clears_a_stale_typed_code() -> None: # A world demoted by a typed `seed_failed` reset failure used to keep that code in # `_down_codes` forever if the reconcile that follows gives up UNTYPED (a bare `OSError`, or diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py index 6ab4fdc0..2000e9cf 100644 --- a/tests/harness/test_process_preflight.py +++ b/tests/harness/test_process_preflight.py @@ -507,6 +507,25 @@ def mutate(body: dict[str, Any]) -> dict[str, Any]: preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) +@pytest.mark.parametrize("colliding_port", [24000, 24099, 25000, 25799]) +def test_a_fixed_port_colliding_with_the_rabbitmq_management_band_is_rejected( + tmp_path: Path, colliding_port: int +) -> None: + """M4: `process_runtime.py`'s own rabbitmq management listener binds at `amqp_port + 10000` + (`_rabbitmq_management_port`) -- a `fixed_port` landing there is not covered by either base + port-formula band, so a bundle could claim a port the harness itself is about to bind. Bands: + 24000-24099 (job-shared amqp shifted) and 25000-25799 (per-world amqp shifted, the one + reachable in practice since rabbitmq's catalog entry is `datadir_copy`-only).""" + + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = colliding_port + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="fixed_port_reserved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + def test_a_fixed_port_outside_both_bands_is_accepted(tmp_path: Path) -> None: def mutate(body: dict[str, Any]) -> dict[str, Any]: body["processes"][1]["fixed_port"] = 9000 diff --git a/tests/harness/test_world_stores_container.py b/tests/harness/test_world_stores_container.py new file mode 100644 index 00000000..e2a1cb21 --- /dev/null +++ b/tests/harness/test_world_stores_container.py @@ -0,0 +1,52 @@ +"""Regression: `ContainerStore._await_ready`'s timeout path must not leak the container it +started. + +`start()` sets `_started = True` (world/stores/container.py) BEFORE `_await_ready()` runs, so a +container that never answers within `READY_TIMEOUT_SECONDS` used to raise `StoreError` straight +out of `start()` with the container still running -- nothing else ever calls `stop()` on a store +whose own `start()` raised, so the container was orphaned for good. Docker-gated, following the +same rule as `tests/test_harness_stores.py`'s own Postgres lane. +""" + +from __future__ import annotations + +import pytest + +from fi.alk.bench._docker import docker_available +from fi.alk.harness.world.stores import container +from fi.alk.harness.world.stores.postgres import PostgresStore + +pg = pytest.mark.skipif(not docker_available(), reason="docker daemon unavailable") + + +class NeverReadyStore(PostgresStore): + """Boots exactly like `PostgresStore` (so the container genuinely starts and stays running -- + a real "running but not an engine that answers" case, not a `docker run` failure) but its + `probe()` never succeeds, so `_await_ready` always times out.""" + + def probe(self) -> None: + raise ConnectionError("deliberately never ready, for the leak-on-timeout regression") + + +def _container_exists(name: str) -> bool: + listed = container.docker( + "ps", "-a", "--filter", f"name=^{name}$", "--format", "{{.Names}}", check=False + ) + return name in listed.splitlines() + + +@pg +def test_await_ready_timeout_removes_the_container_it_started(monkeypatch) -> None: + # A short deadline keeps this fast -- the bug and the fix are both about WHAT HAPPENS on + # timeout, not about how long a real engine takes to boot. + monkeypatch.setattr(container, "READY_TIMEOUT_SECONDS", 0.5) + store = NeverReadyStore() + try: + with pytest.raises(container.StoreError, match="did not answer"): + store.start() + assert not _container_exists(store.container) + assert store._started is False # same post-teardown state `stop()` leaves on success + assert store.port is None + finally: + # Backstop only -- a passing test already removed it via the fixed timeout path. + store.stop() From ca31f43465dc2747c841c035b12ac5fea1a5fd9a Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 26 Aug 2026 00:54:35 +0530 Subject: [PATCH 15/20] =?UTF-8?q?feat(harness):=20scenario-source=20adapte?= =?UTF-8?q?r=20=E2=80=94=20bundle=20documents=20to=20runnable=20scenarios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_entrypoint.py | 31 +- src/fi/alk/harness/scenario_source.py | 361 ++++++++ tests/harness/test_hosted_entrypoint.py | 391 ++++++++- tests/harness/test_scenario_source.py | 1042 +++++++++++++++++++++++ 4 files changed, 1811 insertions(+), 14 deletions(-) create mode 100644 src/fi/alk/harness/scenario_source.py create mode 100644 tests/harness/test_scenario_source.py diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index a9273cd3..abc1ee3c 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -62,6 +62,7 @@ ProcessRuntimeProvider, RuntimeEndpoint, ) +from .scenario_source import BundleScenarioSource, ScenarioDocumentInvalid, bundle_has_scenarios from .world.handle import HostedWorld from .world.stores.postgres import AttachedPostgresStore @@ -256,6 +257,7 @@ async def build( *, pool: WorldPool, world_factory: WorldFactory, + bundle_dir: Path, ) -> Sequence[Scenario]: ... @@ -268,8 +270,9 @@ async def build( *, pool: WorldPool, world_factory: WorldFactory, + bundle_dir: Path, ) -> Sequence[Scenario]: - del job, bundle, scenarios_client, pool, world_factory + del job, bundle, scenarios_client, pool, world_factory, bundle_dir raise ScenarioSourceNotWired( "no ScenarioSource wired -- scenario generation is not implemented in this repo yet " "(Scenario Generation Contract, in review)" @@ -1576,9 +1579,19 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m adapter.stage_changed(HarnessStage.VALIDATING_SCENARIOS) await adapter.aflush_events() world_factory = deps.build_world_factory(work_directory) + # An injected `ScenarioSource` (every test, every future caller) always wins -- the real + # bundle-reading adapter (scenario_source.py) only steps in when the default + # `NotWiredScenarioSource` is still in place AND the bundle actually carries a `scenarios/` + # directory (the LAYOUT DECISION's presence test). A bundle without one keeps the existing + # typed `ScenarioSourceNotWired` failure below -- no regression for a job whose scenarios + # are not generated yet. + scenario_source = deps.scenario_source + if isinstance(scenario_source, NotWiredScenarioSource) and bundle_has_scenarios(bundle_dir): + scenario_source = BundleScenarioSource() try: - scenarios = await deps.scenario_source.build( - job, manifest, scenarios_client, pool=pool, world_factory=world_factory + scenarios = await scenario_source.build( + job, manifest, scenarios_client, pool=pool, world_factory=world_factory, + bundle_dir=bundle_dir, ) except (ob.HostedFencedError, ob.HostedAttemptSupersededError): # `ScenariosClient._post` re-raises these after latching `channel_state` -- a fence @@ -1603,6 +1616,18 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, code="scenario_preallocation_failed", message=str(exc), ) + except ScenarioDocumentInvalid as exc: + # A scenario document that will not even compile is a generation-stage content defect + # (deterministic on retry), never a transport failure -- same rationale as + # `_SCENARIO_ENTRY_INVALID_CODE`'s other use below, reused rather than inventing a new + # code for the same pair of (domain, stage). + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code=_SCENARIO_ENTRY_INVALID_CODE, message=str(exc), + ) # Defense against a malformed scenario entry (K1) reaching the scheduler, which reads # `scenario_key`/`sub_goals`/`setup`/`ready` with plain attribute access and would raise diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py new file mode 100644 index 00000000..3a4192b5 --- /dev/null +++ b/src/fi/alk/harness/scenario_source.py @@ -0,0 +1,361 @@ +"""The scenario-source adapter: reads generated scenario documents out of the bundle (code-as-text, +on the on-disk layout `folder.py` documents) and turns them into the `Scenario`/`SubGoal` objects +`hosted_scheduler.py` actually drives. + +Deliberately does NOT import `fi.alk.harness.folder` or `fi.alk.harness.scenario` for the model: +both exist at HEAD, but HEAD's `Scenario` carries no `scenario_key`/`scenario_id` (those are +pr63-only) and its default `extra="ignore"` would silently discard exactly the two fields the +scheduler needs off a `scenario.json` written in the newer shape. So this module reads +`scenario.json` as a plain dict and pulls fields out by key, mirroring the documented layout +instead of depending on either model -- see the report's design-decisions section for the +consequences of that choice (HEAD-model drift). + +Karthik's Scenario Generation Contract (the `provision`/`begin` wire shapes) has not landed. This +module builds the bundle-reading + compiling + wrapping side only; `register_with_platform` below +is the one seam a later change wires in once that contract exists. +""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Sequence + +if TYPE_CHECKING: + from .hosted_entrypoint import ScenariosClient + +# LAYOUT DECISION (contract-silent -- hosted-execution-seams.md v1.15 §2 never mentions scenario +# documents, and §7 assigns the on-disk layout to Karthik's contract, status "in review"). Scenario +# documents live at `///...`, matching `folder.py`'s own +# `SCENARIOS` constant, so a write_folder destination of `` lands correctly with no +# translation. Kept as one module-level constant so a later contract can move it in one edit. +SCENARIOS_DIRNAME = "scenarios" + +_CHECKS_DIRNAME = "checks" +_SCENARIO_JSON = "scenario.json" +_SETUP_PY = "setup.py" +_READY_PY = "ready.py" + +# R1-5: divergence (a) (compile once, at load) moves every scenario file's compile+exec into +# `asyncio.to_thread(load_scenarios, ...)`, OUTSIDE every phase budget the scheduler enforces +# (`SETUP_TIMEOUT_SECONDS`/`READY_TIMEOUT_SECONDS`/`CHECK_TIMEOUT_SECONDS` all apply downstream, to +# `_run_phase`). Pathological module-level code (`while True: pass`, a blocking socket read) would +# otherwise hang the load with zero terminal events, no timeout catching it, ever. One generous +# wall-clock budget here converts that hang into a typed terminal instead -- a worker thread cannot +# actually be killed, so this accepts a leaked thread over an unbounded one, on the reasoning that a +# terminal event today is strictly better than none ever. +_LOAD_TIMEOUT_SECONDS = 60.0 + +# The TEXT of a judged sub-goal's check is never persisted by `folder.py`'s `write_folder` (only +# `SubGoal.deterministic()` entries get a `checks/.py` file) -- this fixed marker stands in +# for it so `SubGoal.judged` (mandatory; read by plain attribute access in `hosted_scheduler.py`) +# is never empty for a sub-goal this reader knows is judged. The round-trip loss this represents is +# recorded under CONTRACT QUESTIONS in the report. +_JUDGED_MARKER = "judged (reason not persisted by folder.py's on-disk layout)" + + +class ScenarioDocumentInvalid(RuntimeError): + """A scenario folder under `/scenarios/` is unreadable or malformed: missing + `scenario.json`, invalid JSON, a field of the wrong shape, or a `setup.py`/`ready.py`/ + `checks/.py` that will not compile. Raised rather than skipped -- `folder.py`'s own + `read_all` swallows exactly this and continues, which is right for a human editing a suite by + hand and wrong for a hosted job, where a bad scenario silently vanishing from the run reads as + a suite that passed with fewer scenarios than it should have.""" + + +def bundle_has_scenarios(bundle_dir: Path) -> bool: + """The LAYOUT DECISION's presence test: `/scenarios/` exists and at least one of + its subdirectories holds a `scenario.json`. Deliberately narrow -- an empty or missing + `scenarios/` must not flip the wiring decision away from the safe `NotWiredScenarioSource` + default. A bundle that means to carry scenarios but got the layout wrong fails loudly once + `load_scenarios` actually reads it, not silently by being treated as scenario-free here. + + An unreadable `scenarios/` directory (permission denied, race with deletion, etc.) reports + False rather than raising `OSError` (R1-1): this call sits in `hosted_entrypoint.py`'s wiring + `if` BEFORE the `try`/`except` that maps `ScenarioDocumentInvalid` to a typed terminal, so an + escape here would kill the whole guest process with no terminal event at all. Falling back to + `NotWiredScenarioSource`'s existing typed failure is the safe direction -- the alternative of + raising here has nowhere typed to land. + """ + root = bundle_dir / SCENARIOS_DIRNAME + if not root.is_dir(): + return False + try: + children = list(root.iterdir()) + except OSError: + return False + return any((child / _SCENARIO_JSON).is_file() for child in children if child.is_dir()) + + +def _judged_placeholder_check(world: Any, calls: Any) -> None: + """The check for a judged (non-deterministic) sub-goal: always reports "held", via the same + convention a deterministic check uses. `SubGoalResult.judged` -- not this return value -- is + what has to tell downstream a real judge still has to run; see CONTRACT QUESTIONS in the + report for the gap that leaves (a judged-only scenario reports "passed" before any judge runs). + """ + del world, calls + return None + + +def _compile_entry( + source: str, *, label: str, entry: str, allow_empty: bool = True +) -> Callable[..., object]: + """One scenario code-text -> a bare callable that raises, compiled ONCE here rather than per + call. Mirrors `folder.py`'s `_run` in exactly two respects: `compile(source, name, "exec")` + into a fresh, empty-dict namespace with default builtins, and (when `allow_empty`) + empty/whitespace-only source is a no-op success. Deliberately diverges from `_run` in the two + respects the brief calls out: + (a) compiling here, at load, turns a syntax error into one typed terminal for the whole job + instead of a per-scenario fault discovered mid-run; (b) the compiled function is returned + BARE, never wrapped in `_run`'s `Outcome` -- `hosted_scheduler.py`'s `_run_phase` is what + classifies a raised exception into `setup_crashed`/`ready_broken`/`check_broken`/a timeout, and + an `Outcome` return here would swallow every one of those before `_run_phase` ever saw it. + `_run`'s complaint-sentence return convention (a non-None, non-True, non-empty-string value + means "did not hold") is left untranslated for the same reason: `hosted_scheduler.py`'s own + `_classify_ready`/`_classify_check` already own that classification on the return-VALUE side of + this boundary; only the raise-vs-return boundary belongs to this module. + + `allow_empty=False` is for `check` entries only (R1-2): an EXISTING `checks/.py` that is + empty or whitespace-only compiles to nothing, and handing back a no-op "held" callable -- the + right behavior for "no setup/ready code here" -- would silently turn "there is no check" into + "the check passed": a vacuous deterministic pass, which is exactly what `hosted_scheduler.py` + forbids one scenario level up ("scenario declared zero sub_goals"). Absence of the file + entirely is what means "judged" (see `_load_one`); an existing-but-empty file is malformed. + """ + if not source.strip(): + if allow_empty: + return lambda *args: None + raise ScenarioDocumentInvalid(f"{label} defines no {entry}()") + try: + code = compile(source, f"<{label}>", "exec") + except (SyntaxError, ValueError) as exc: + # SyntaxError is the common case; a NUL byte in the source raises ValueError on some + # interpreter versions (R1-1) rather than SyntaxError -- both are the same content defect. + raise ScenarioDocumentInvalid(f"{label} would not compile: {exc}") from exc + namespace: dict[str, Any] = {} + try: + exec(code, namespace) # noqa: S102 - scenario code is meant to be exec'd; see CONTRACT QUESTIONS + except (Exception, SystemExit, KeyboardInterrupt) as exc: # noqa: BLE001 - see R1-1 + # A module-level `sys.exit()`/`raise SystemExit(...)` in the file itself is a malformed + # document, not a request to shut the guest process down -- `SystemExit`/`KeyboardInterrupt` + # are `BaseException`, not `Exception`, so a bare `except Exception` (the pre-R1-1 shape) + # let them straight through this boundary and out of `run_job` with zero terminal events, + # the guest exiting with whatever code the scenario file itself chose. + raise ScenarioDocumentInvalid(f"{label} would not compile: {exc}") from exc + function = namespace.get(entry) + if not callable(function): + raise ScenarioDocumentInvalid(f"{label} defines no {entry}()") + return function + + +@dataclass(frozen=True) +class _CompiledSubGoal: + """Satisfies `hosted_scheduler.SubGoal`: `name`/`judged` as plain attributes, `check` as a bare + callable. Stored as instance DATA rather than a `def check(self, world, calls)` method so + `goal.check(world, calls)` invokes the compiled function directly with exactly the two + positional arguments `_run_phase` passes -- a real method would prepend `self` as a third.""" + + name: str + judged: str + check: Callable[[Any, Any], object] + + +@dataclass(frozen=True) +class _CompiledScenario: + """Satisfies `hosted_scheduler.Scenario`. `scenario_key`/`scenario_id` are carried VERBATIM + from the document, including an empty `scenario_id` -- synthesizing one here would hide that + pre-allocation has not actually run (see CONTRACT QUESTIONS: receipts carry `scenario_id ""` + until that seam is wired). `setup`/`ready` are likewise stored as data, for the same reason as + `_CompiledSubGoal.check` above.""" + + scenario_key: str + scenario_id: str + sub_goals: tuple[_CompiledSubGoal, ...] + setup: Callable[[Any], object] + ready: Callable[[Any], object] + + +def _read_text(path: Path, *, label: str) -> str: + """Missing is "" (mirrors `folder.py`'s own missing-setup/ready-is-empty convention); present + but unreadable (permission denied, a directory instead of a file) or present but not valid + UTF-8 is a typed `ScenarioDocumentInvalid`, never a raw `OSError`/`UnicodeDecodeError` escaping + this module (R1-1) -- both are equally "this scenario folder is malformed", the same + conclusion `_load_one`'s other reads already reach for a bad `scenario.json`. + """ + if not path.exists(): + return "" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise ScenarioDocumentInvalid(f"{label}: cannot read {path.name}: {exc}") from exc + + +def _validate_subgoal_name(name: str, *, folder_name: str) -> None: + """R1-3: `sub_goals[]` entries are used verbatim to build `checks/.py` -- a path + separator or a `..` segment lets a name escape `checks/` (and the sealed bundle) entirely: an + absolute name execs an arbitrary file never hashed into the bundle's `files[]` (bypassing the + §2e integrity seal), and a `../`-style traversal that resolves to nothing silently turns into a + JUDGED sub-goal (`check_path.is_file()` is False) instead of a typed failure. Rejecting + anything but a plain filename component closes both.""" + if not name or "/" in name or "\\" in name or name in (".", ".."): + raise ScenarioDocumentInvalid( + f"{folder_name}: sub_goals name {name!r} is not a plain filename " + "(no path separators, no '..', no leading '/')" + ) + + +def _load_one(folder: Path) -> _CompiledScenario: + """One scenario folder -> a `Scenario`-protocol object. Mirrors `folder.py`'s documented + layout (`scenario.json` + `setup.py` + `ready.py` + `checks/.py`) but reads + `scenario.json` itself as a plain dict rather than through `fi.alk.harness.scenario.Scenario` + -- see the module docstring. `folder.py`'s `read_folder` restores only `setup_code`/ + `ready_code` from a folder; it does not read `checks/` at all, so every `checks/.py` for + each name in the document's `sub_goals` is read here, by this module, directly. + """ + body_path = folder / _SCENARIO_JSON + try: + raw = body_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # UnicodeDecodeError is a ValueError, not an OSError -- widened alongside it (R1-1) so a + # non-UTF-8 `scenario.json` is the same typed failure as an unreadable one, not an escape. + raise ScenarioDocumentInvalid( + f"{folder.name}: cannot read {_SCENARIO_JSON}: {exc}" + ) from exc + try: + body = json.loads(raw) + except json.JSONDecodeError as exc: + raise ScenarioDocumentInvalid( + f"{folder.name}: {_SCENARIO_JSON} is not valid JSON: {exc}" + ) from exc + if not isinstance(body, dict): + raise ScenarioDocumentInvalid(f"{folder.name}: {_SCENARIO_JSON} is not a JSON object") + + scenario_key = body.get("scenario_key", "") + if not isinstance(scenario_key, str): + raise ScenarioDocumentInvalid(f"{folder.name}: scenario_key is not a string") + scenario_id = body.get("scenario_id", "") + if not isinstance(scenario_id, str): + raise ScenarioDocumentInvalid(f"{folder.name}: scenario_id is not a string") + sub_goal_names = body.get("sub_goals", []) + if not isinstance(sub_goal_names, list) or not all( + isinstance(name, str) for name in sub_goal_names + ): + raise ScenarioDocumentInvalid(f"{folder.name}: sub_goals is not a list of strings") + for name in sub_goal_names: + _validate_subgoal_name(name, folder_name=folder.name) + + setup_code = _read_text(folder / _SETUP_PY, label=folder.name) + ready_code = _read_text(folder / _READY_PY, label=folder.name) + setup = _compile_entry(setup_code, label=f"{folder.name}/{_SETUP_PY}", entry="setup") + ready = _compile_entry(ready_code, label=f"{folder.name}/{_READY_PY}", entry="ready") + + sub_goals: list[_CompiledSubGoal] = [] + for name in sub_goal_names: + check_path = folder / _CHECKS_DIRNAME / f"{name}.py" + if check_path.is_file(): + check_code = _read_text(check_path, label=folder.name) + check = _compile_entry( + check_code, label=f"{folder.name}/{_CHECKS_DIRNAME}/{name}.py", entry="check", + allow_empty=False, # R1-2: an existing-but-empty check file is invalid, never a + # vacuous pass -- absence of the file is what means "judged". + ) + judged = "" + else: + # No `checks/.py` -- per `write_folder`'s own `deterministic()` filter, this + # name is a JUDGED sub-goal. + judged = _JUDGED_MARKER + check = _judged_placeholder_check + sub_goals.append(_CompiledSubGoal(name=name, judged=judged, check=check)) + + return _CompiledScenario( + scenario_key=scenario_key, + scenario_id=scenario_id, + sub_goals=tuple(sub_goals), + setup=setup, + ready=ready, + ) + + +def load_scenarios(bundle_dir: Path) -> list[_CompiledScenario]: + """Every scenario document under `/scenarios/`, compiled and wrapped, in the same + sorted-by-folder-name order `folder.py`'s `read_all` uses. Raises `ScenarioDocumentInvalid` on + the FIRST unreadable or malformed folder -- unlike `read_all`, which skips one and continues; + a hosted job has nobody watching a suite by hand to notice a scenario silently missing from the + count, so a folder this reader cannot use fails the whole job instead of shrinking it quietly. + """ + root = bundle_dir / SCENARIOS_DIRNAME + if not root.is_dir(): + raise ScenarioDocumentInvalid(f"{root} is not a directory") + try: + entries = sorted(root.iterdir()) + except OSError as exc: + # An unreadable `scenarios/` directory is the same typed failure as any other malformed + # document (R1-1) -- this is inside `run_job`'s `try`/`except ScenarioDocumentInvalid` + # (unlike `bundle_has_scenarios`'s own guard above), so raising here is the safe direction. + raise ScenarioDocumentInvalid(f"{root}: cannot list scenario folders: {exc}") from exc + scenarios: list[_CompiledScenario] = [] + for folder in entries: + if not folder.is_dir(): + continue + scenarios.append(_load_one(folder)) + if not scenarios: + raise ScenarioDocumentInvalid(f"{root} contains no scenario folders") + return scenarios + + +class BundleScenarioSource: + """The real `ScenarioSource`: reads and compiles the bundle's own scenario documents. + `hosted_entrypoint.run_job` wires this in only when the injected source is still the default + `NotWiredScenarioSource` AND the bundle actually carries a `scenarios/` directory (the LAYOUT + DECISION's presence test) -- an injected `ScenarioSource` (every test, every future caller) + always wins over this one. + """ + + async def build( + self, + job: Any, + bundle: Any, + scenarios_client: "ScenariosClient", + *, + pool: Any, + world_factory: Any, + bundle_dir: Path, + ) -> Sequence[_CompiledScenario]: + del job, bundle, scenarios_client, pool, world_factory + # `Path.read_text`/`iterdir`/`compile` are all blocking filesystem+CPU work -- run off the + # event loop the same way `hosted_entrypoint.py` already does for `bundle_source.load` and + # `preflight_bundle`, rather than stalling every other in-flight scenario behind it. + try: + return await asyncio.wait_for( + asyncio.to_thread(load_scenarios, bundle_dir), timeout=_LOAD_TIMEOUT_SECONDS + ) + except asyncio.TimeoutError as exc: + # R1-5: the underlying thread cannot actually be canceled/killed -- it is left running + # in the background. Converting the hang into a typed terminal here is still strictly + # better than the pre-fix behavior (no terminal event, ever): the job gets an honest, + # bounded FAILED verdict instead of hanging until the platform's own wall clock gives up. + raise ScenarioDocumentInvalid( + f"{bundle_dir / SCENARIOS_DIRNAME}: loading scenario documents exceeded " + f"{_LOAD_TIMEOUT_SECONDS:.0f}s" + ) from exc + + +async def register_with_platform( + scenarios_client: "ScenariosClient", scenarios: Sequence[_CompiledScenario] +) -> Sequence[_CompiledScenario]: + """SEAM -- not called anywhere in this module, and not wired into `BundleScenarioSource.build` + above. Once scenario-generation-contract.md section 3 publishes the `provision`/`begin` payload + and response shapes, this is where they get built from `scenarios` and posted through + `scenarios_client.provision(...)`/`.begin(...)`, and where the platform-assigned `scenario_id`s + that `provision` returns get merged back onto each scenario before `build` hands the list to + the scheduler. Left unimplemented rather than guessing a body the contract has not published + (CONTRACT GAP) -- wiring this in is the one remaining integration step this module cannot + finish alone. + """ + del scenarios_client, scenarios + raise NotImplementedError( + "register_with_platform: scenario-generation-contract.md section 3 (the provision/begin " + "wire shapes) has not landed -- this seam is intentionally left unwired" + ) diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 19a6e7db..450470e0 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -21,6 +21,7 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any, Callable +from unittest import mock from fi.alk.harness import hosted_entrypoint as he from fi.alk.harness import outbound as ob @@ -59,6 +60,7 @@ RuntimeEndpoint, RuntimeState, ) +from fi.alk.harness.scenario_source import SCENARIOS_DIRNAME from fi.simulate.runtime.spec import RuntimeRequirements, SecretRef SCHEMA_SQL = b"CREATE TABLE riders (id int);\n" @@ -464,9 +466,12 @@ def __init__(self, scenarios: list[FakeScenario]) -> None: async def build( self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, - world_factory: Any, + world_factory: Any, bundle_dir: Path, ) -> list[FakeScenario]: - del job, bundle, pool, world_factory + # `bundle_dir` (p12: the scenario-source adapter's own seam) is unused by this in-memory + # fake -- accepted only because `run_job` now forwards it to every `ScenarioSource.build`, + # injected or not. + del job, bundle, pool, world_factory, bundle_dir await asyncio.to_thread( scenarios_client.provision, {"scenario_keys": [s.scenario_key for s in self._scenarios]}, @@ -490,6 +495,7 @@ class Harness: transport: FakeTransport provisioner: FakeProvisioner deps: he.HostedEntrypointDeps + bundle_dir: Path def _build_harness( @@ -506,6 +512,14 @@ def _build_harness( parallelism: int = 1, build_output: dict[str, Any] | None = None, artifacts: HarnessArtifactPolicy | None = None, + # p12: leaves `deps.scenario_source` at its real default (`NotWiredScenarioSource`) instead of + # the usual `FakeScenarioSource` -- for the scenario_source.py wiring tests, which need + # `run_job` to actually choose between the default and the real `BundleScenarioSource` itself. + use_default_scenario_source: bool = False, + # p12: swaps in `_write_bundle_with_scenario_files` for the scenario-source wiring tests, which + # need real scenario documents hashed into the manifest's `files[]` (§2e `bundle_file_unlisted` + # otherwise rejects them at preflight) -- every other caller keeps the plain `_write_bundle`. + bundle_writer: Callable[[Path], Any] = _write_bundle, ) -> Harness: tmp = Path(tempfile.mkdtemp(prefix="p10-e2e-")) work = tmp / "work" @@ -513,7 +527,7 @@ def _build_harness( output = work / "artifacts" bundle_dir = work / he.DEFAULT_BUNDLE_DIR_NAME source.mkdir(parents=True, exist_ok=True) - _write_bundle(bundle_dir) + bundle_writer(bundle_dir) if corrupt_bundle is not None: corrupt_bundle(bundle_dir) @@ -548,7 +562,9 @@ def build_call_runner(adapter: he.OutboundAdapter) -> FakeCallRunner: deps = he.HostedEntrypointDeps( load_capabilities=lambda: capabilities, bundle_source=he.DefaultBundleSource(), - scenario_source=FakeScenarioSource(scenarios), + scenario_source=( + he.NotWiredScenarioSource() if use_default_scenario_source else FakeScenarioSource(scenarios) + ), build_transport=lambda: transport, build_provider=lambda: provisioner, build_call_runner=build_call_runner, @@ -560,7 +576,7 @@ def build_call_runner(adapter: he.OutboundAdapter) -> FakeCallRunner: ) return Harness( tmp=tmp, work=work, source=source, output=output, job_path=job_path, transport=transport, - provisioner=provisioner, deps=deps, + provisioner=provisioner, deps=deps, bundle_dir=bundle_dir, ) @@ -1250,8 +1266,8 @@ def test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped() -> # the `finally` with no explicit close anywhere on this path -- only the backstop can close it. async def scenario() -> None: class ExplodingScenarioSource: - async def build(self, job, bundle, scenarios_client, *, pool, world_factory): - del job, bundle, scenarios_client, pool, world_factory + async def build(self, job, bundle, scenarios_client, *, pool, world_factory, bundle_dir): + del job, bundle, scenarios_client, pool, world_factory, bundle_dir raise MemoryError("boom") harness = _build_harness(scenarios=[], instances=1) @@ -1329,9 +1345,9 @@ def __init__(self, scenarios: list[Any]) -> None: async def build( self, job: Any, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, - world_factory: Any, + world_factory: Any, bundle_dir: Path, ) -> list[Any]: - del job, bundle, scenarios_client, pool, world_factory + del job, bundle, scenarios_client, pool, world_factory, bundle_dir return self._scenarios async def scenario() -> None: @@ -2005,9 +2021,9 @@ def __init__(self, *, chatter_count: int) -> None: async def build( self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, - pool: Any, world_factory: Any, + pool: Any, world_factory: Any, bundle_dir: Path, ) -> list[FakeScenario]: - del job, bundle, scenarios_client, world_factory + del job, bundle, scenarios_client, world_factory, bundle_dir adapter = pool._outbound # no adapter seam on ScenarioSource itself for i in range(self._chatter_count): await adapter.log(level="info", message=f"pre-run chatter {i}") @@ -2163,6 +2179,350 @@ async def slow_emit(self: HostedScheduler, result: RunResult) -> None: asyncio.run(scenario()) +# ================================================================================================= +# p12: scenario_source.py wiring -- item 4. Writes real scenario documents (`folder.py`'s own +# on-disk layout) straight into `harness.bundle_dir`, matching `scenario_source.py`'s +# `SCENARIOS_DIRNAME` constant. Deliberately its own tiny writer rather than importing +# `test_scenario_source.py`'s `_write_scenario` (no cross-test-module private-helper imports). +# +# §2e preflight (`bundle_file_unlisted`) rejects any bundle file absent from the manifest's +# `files[]` -- exactly the CONTRACT QUESTIONS obligation the report calls out for a real Scenario +# Generation Contract bundle author. So a scenario-bearing bundle for these tests cannot just drop +# files under `bundle_dir/scenarios/` after `_write_bundle` has already sealed the manifest; the +# scenario files have to be hashed into `files[]` and the bundle re-sealed, the same way +# `test_process_preflight.py`'s own `_build_bundle(extra_files=...)` does it. +# ================================================================================================= + + +def _scenario_doc_files( + name: str, *, scenario_key: str, scenario_id: str = "", sub_goals: list[str] | None = None, + setup_code: str = "", checks: dict[str, str] | None = None, +) -> dict[str, bytes]: + """One scenario folder's contents as {relative path: bytes} -- fed to + `_write_bundle_with_scenario_files` below rather than written straight to disk, so every byte + can be hashed into the manifest's `files[]` before the bundle is sealed.""" + body = { + "name": name, "scenario_key": scenario_key, "scenario_id": scenario_id, + "sub_goals": sub_goals or [], + } + prefix = f"{SCENARIOS_DIRNAME}/{name}" + files = {f"{prefix}/scenario.json": json.dumps(body).encode("utf-8")} + if setup_code: + files[f"{prefix}/setup.py"] = setup_code.encode("utf-8") + for goal_name, code in (checks or {}).items(): + files[f"{prefix}/checks/{goal_name}.py"] = code.encode("utf-8") + return files + + +def _write_bundle_with_scenario_files(root: Path, scenario_files: dict[str, bytes]) -> EnvironmentBundleV2: + """`_write_bundle`, plus `scenario_files` hashed into `files[]` and the digest re-sealed over + all of it -- otherwise every one of these trips `bundle_file_unlisted` at preflight, before + `scenario_source.build()` is ever reached.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL, **scenario_files} + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", "migrations": ["db/schema.sql"], "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +def test_bundle_without_scenarios_keeps_the_notwired_regression() -> None: + # item 4: a bundle that does not carry a `scenarios/` directory must keep behaving exactly as + # it did before this adapter existed -- the default `NotWiredScenarioSource`'s typed failure, + # never a crash and never a silently-empty run. + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1, use_default_scenario_source=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "platform_sync" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() -> None: + # item 4 + 5c (end to end): the presence test flips the default over to the real + # `BundleScenarioSource` -- no `FakeScenarioSource` involved anywhere in this test. One + # deterministic sub_goal that genuinely holds against the fake world, so this proves a real + # COMPLETED pass, not just that the vacuous-pass guard fired. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + # `scenario_id` non-empty here on purpose -- see + # `test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema` just below for + # the (newly discovered, load-bearing) reason an EMPTY one cannot be used to + # prove a receipt actually arrives. + "passing", scenario_key="passing", scenario_id="platform-passing", + sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["failure"] is None + + # terminal last: the terminal record is the final event this run ever pushed. + assert harness.transport.event_records[-1].get("type") == "terminal" + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed"} # the real scheduler actually ran it, and it held + + asyncio.run(scenario()) + + +def test_empty_scenario_key_from_bundle_document_fails_cleanly_via_existing_validation() -> None: + # Mutation table item 2: "empty-key fixture -> typed failure", at the FULL integration level -- + # a hand-written document with an empty `scenario_key` flows verbatim through this adapter + # (work item 3: never synthesized) into the scheduler's OWN pre-existing defense + # (`_validate_scenario_entry`, untouched by this task), which must catch it as a typed FAILED + # terminal rather than an `AttributeError` deep in the scheduler. `test_scenario_source.py` + # covers the reader's verbatim carry and the mutation that would break it; this is the + # end-to-end proof that the two halves agree. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, _scenario_doc_files("s1", scenario_key="", sub_goals=[]) + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "environment" + assert failure["code"] == "scenario_preallocation_failed" + assert "scenario_key" in failure["message"] + + asyncio.run(scenario()) + + +def test_mutation_adapter_off_makes_the_e2e_test_fail() -> None: + # Mutation table item 1: "adapter-off -> e2e test fails". Patches `bundle_has_scenarios` (as + # seen from `hosted_entrypoint.py`'s own namespace, where it was imported) to always report + # "no scenarios here" -- simulating hard-rule edit (b) never having been made, i.e. the wiring + # `if isinstance(...) and bundle_has_scenarios(...)` guard permanently failing closed. + # + # R1-6 fold-in (p12-review-r1.md LOW finding): the original version of this test asserted the + # MUTANT's own failure terminal directly -- true, but it never actually ran the `stage == + # "completed"` assertion that is the real kill. This version runs the REAL + # `test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present` under the + # patch and checks THAT it fails. `mock.patch.object` restores the original function in its own + # `finally` regardless of how the inner call ends -- no manual restore bookkeeping needed. + with mock.patch.object(he, "bundle_has_scenarios", lambda bundle_dir: False): + try: + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() + except AssertionError: + pass + else: + raise AssertionError( + "adapter-off mutant did not fail " + "test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present " + "(no pytest.raises here -- this file runs stand-alone via TESTS, per its own " + "module docstring)" + ) + + # Restored: the real test passes again. + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() + + +def test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema() -> None: + # NEWLY DISCOVERED while writing the test above: `outbound.py`'s `ResultReceiptDraft` schema + # requires `scenario_id` to be non-empty (pydantic `min_length=1`). The brief mandates carrying + # `scenario_id` VERBATIM off the document -- including empty, until pre-allocation is wired -- + # so a scenario whose pre-allocation has not run gets its receipt rejected at construction, + # logged as an error, and DROPPED, while the job still reports COMPLETED with that scenario + # counted as `passed`/`failed` in `scenario_counts`. This sharpens the brief's own "blocking + # integration obligation" from a documentation concern into a concrete, verified one: today, a + # bundle-sourced scenario can never actually deliver a receipt to the platform until pre- + # allocation assigns it a real `scenario_id`. See CONTRACT QUESTIONS in the report. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "passing", scenario_key="passing", scenario_id="", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" # the job itself does not fail + assert payload["scenario_counts"]["passed"] == 1 # ...and reports the scenario as passed... + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {} # ...yet no receipt for it ever reached the platform. + + error_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and record["payload"].get("level") == "error" + and "ResultReceiptDraft" in record["payload"].get("message", "") + ] + assert len(error_logs) == 1 # the drop is at least loud, not silent -- but still a drop. + + asyncio.run(scenario()) + + +def test_injected_scenario_source_always_wins_over_the_bundle_adapter() -> None: + # item 4: even when the bundle ALSO carries a valid `scenarios/` directory, an explicitly + # injected `ScenarioSource` must be used untouched -- the presence test only ever applies to + # the untouched default. + async def scenario() -> None: + injected = [FakeScenario("from-fake", "platform-from-fake", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=injected, instances=1, # FakeScenarioSource, as usual -- not the default + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, _scenario_doc_files("from-bundle", scenario_key="from-bundle", sub_goals=[]) + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"from-fake": "passed"} # the bundle's own scenario never ran + + asyncio.run(scenario()) + + +# ================================================================================================= +# R1-1 (CRITICAL, p12-review-r1.md) -- through the REAL `run_job`, on a sealed, preflight-clean +# bundle: a scenario document must never be able to set the guest's own exit code, and an unreadable +# or malformed scenario file must never escape as an unhandled exception with zero terminal events. +# The chmod-000 "unreadable file" trigger is proven directly against `scenario_source.py` in +# `test_scenario_source.py` instead of here -- see that file's R1-1 section docstring for why the +# full bundle/preflight pipeline cannot exercise it without touching `process_preflight.py`. +# ================================================================================================= + + +def test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure() -> None: + # Before the fix: `sys.exit(0)` at module level inside `setup.py` propagated as a raw + # `SystemExit` straight out of `run_job` -- the guest process itself would exit 0 with ZERO + # terminal events (a "clean terminal that never happened", per spine §0.6), on a bundle whose + # every byte was hashed into the manifest and §2e preflight passed. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "s1", scenario_key="s1", sub_goals=[], setup_code="import sys\nsys.exit(0)\n", + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + # The GUEST's own exit code -- EXIT_OK means "a terminal was reached and flushed", the + # same meaning it carries for any other typed FAILED terminal, never the scenario's own + # sys.exit(0) leaking through `run_job`'s return value. + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # a terminal event WAS delivered -- not an empty event stream + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + failure = payload["failure"] + assert failure["domain"] == "environment" + assert failure["stage"] == "validating_scenarios" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_module_level_sys_exit_three_in_setup_does_not_hijack_the_guests_exit_code() -> None: + # EXIT_FENCED == 3: before the fix, `sys.exit(3)` here was indistinguishable from the guest + # itself choosing to exit fenced -- the platform would read an ordinary scenario content defect + # as a fenced/superseded attempt instead. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "s1", scenario_key="s1", sub_goals=[], setup_code="import sys\nsys.exit(3)\n", + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert code != he.EXIT_FENCED # explicit: the scenario's own exit code did not leak through + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == "environment" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_non_utf8_setup_file_is_contained_as_a_typed_failure_not_an_escape() -> None: + # Before the fix: a raw `UnicodeDecodeError` from `_read_text`'s `.read_text(encoding="utf-8")` + # propagated straight out of `run_job` -- per spine §0.6 a non-zero exit with no terminal event + # reads as `infrastructure` and gets retried to exhaustion, even though this is a deterministic + # content defect that will never succeed on retry. The non-UTF-8 bytes hash and seal fine (§2e + # preflight only hashes raw bytes, never decodes) -- only this module's own `.read_text()` call + # ever attempts to decode them. + async def scenario() -> None: + files = _scenario_doc_files("s1", scenario_key="s1", sub_goals=[]) + files[f"{SCENARIOS_DIRNAME}/s1/setup.py"] = b"def setup(world):\n return '\xff\xfe'\n" + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files(bundle_dir, files), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == "environment" + assert failure["stage"] == "validating_scenarios" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + TESTS = [ test_resolve_parallelism_reads_the_raw_value_without_clamping, test_out_of_range_parallelism_is_rejected_by_preflight_not_clamped, @@ -2214,6 +2574,15 @@ async def slow_emit(self: HostedScheduler, result: RunResult) -> None: test_drain_alone_delivers_a_pre_run_backlog_larger_than_one_batch, test_flush_terminal_alone_must_deliver_the_terminal_before_a_skipped_receipt_under_backlog, test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window, + test_bundle_without_scenarios_keeps_the_notwired_regression, + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present, + test_empty_scenario_key_from_bundle_document_fails_cleanly_via_existing_validation, + test_mutation_adapter_off_makes_the_e2e_test_fail, + test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema, + test_injected_scenario_source_always_wins_over_the_bundle_adapter, + test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure, + test_module_level_sys_exit_three_in_setup_does_not_hijack_the_guests_exit_code, + test_non_utf8_setup_file_is_contained_as_a_typed_failure_not_an_escape, ] diff --git a/tests/harness/test_scenario_source.py b/tests/harness/test_scenario_source.py new file mode 100644 index 00000000..718baf4c --- /dev/null +++ b/tests/harness/test_scenario_source.py @@ -0,0 +1,1042 @@ +"""`scenario_source.py` -- the reader/compiler/wrapper that turns a bundle's own scenario documents +(the on-disk layout `folder.py` documents: `scenarios//scenario.json` + `setup.py` + +`ready.py` + `checks/.py`) into `hosted_scheduler.py`'s `Scenario`/`SubGoal` objects. + +DUPLICATION DISCLOSURE: this file writes its own scenario-folder fixtures (`_write_scenario`) -- +nothing else in the suite writes this layout, so there is nothing to import instead. The +lightweight `WorldPool`/`HostedScheduler` harness below (`_FakeWorld`, `_FakeWorldFactory`, +`_FakeCallRunner`, `_pool`) is adapted from `tests/harness/test_hosted_scheduler.py`'s own fakes of +the same names (not imported across test modules, per the brief -- copied and trimmed to what this +file needs; `_FakeWorld.read_only()` deliberately SHARES its `rows` dict with the writable world, +unlike that file's `InMemoryWorld.read_only()`, because the consumer-proof test below needs a +check to see what setup actually wrote). No `EnvironmentBundleV2` manifest-writing helper (the +`_build_bundle` pattern in `test_process_preflight.py`) is needed anywhere in this file: neither +`load_scenarios` nor `WorldPool`/`HostedScheduler` reads or validates one. + +`asyncio.run` drives every `async def` seam here, matching every other file in this suite (no +pytest-asyncio dependency in this repo). Verified via `pytest tests/harness/test_scenario_source.py`. +""" + +from __future__ import annotations + +import asyncio +import json +import random +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from unittest import mock + +import pytest + +from fi.alk.harness import scenario_source as ss +from fi.alk.harness.hosted_scheduler import Call, CallOutcome, HostedScheduler, WorldPool +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState + +# ================================================================================================= +# Scenario-folder fixture writer -- `folder.py`'s documented layout, hand-written (never through +# `fi.alk.harness.folder`/`fi.alk.harness.scenario`, matching the module under test). +# ================================================================================================= + + +def _write_scenario( + scenarios_root: Path, + name: str, + *, + scenario_key: str = "", + scenario_id: str = "", + sub_goals: list[str] | None = None, + setup_code: str = "", + ready_code: str = "", + checks: dict[str, str] | None = None, + raw_body: dict[str, Any] | None = None, + write_body: bool = True, +) -> Path: + """One scenario folder, written by hand -- deliberately not through pr63's `Scenario` model + (which backfills an empty `scenario_key` via a validator, so a model instance can never + produce the empty-key fixture the brief requires).""" + folder = scenarios_root / name + (folder / "checks").mkdir(parents=True, exist_ok=True) + if write_body: + body = ( + raw_body + if raw_body is not None + else { + "name": name, + "scenario_key": scenario_key, + "scenario_id": scenario_id, + "sub_goals": sub_goals or [], + } + ) + (folder / "scenario.json").write_text(json.dumps(body), encoding="utf-8") + if setup_code: + (folder / "setup.py").write_text(setup_code, encoding="utf-8") + if ready_code: + (folder / "ready.py").write_text(ready_code, encoding="utf-8") + for goal_name, code in (checks or {}).items(): + (folder / "checks" / f"{goal_name}.py").write_text(code, encoding="utf-8") + return folder + + +# ================================================================================================= +# bundle_has_scenarios -- the LAYOUT DECISION's presence test. +# ================================================================================================= + + +def test_bundle_has_scenarios_false_when_no_scenarios_directory(tmp_path: Path) -> None: + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_false_when_scenarios_directory_is_empty(tmp_path: Path) -> None: + (tmp_path / ss.SCENARIOS_DIRNAME).mkdir() + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_false_when_subdirectory_has_no_scenario_json(tmp_path: Path) -> None: + (tmp_path / ss.SCENARIOS_DIRNAME / "s1").mkdir(parents=True) + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_true_with_one_valid_folder(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[]) + assert ss.bundle_has_scenarios(tmp_path) is True + + +# ================================================================================================= +# Reader -- work item 1. +# ================================================================================================= + + +def test_load_scenarios_reads_the_documented_on_disk_layout(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, + "book_a_ride", + scenario_key="book_a_ride", + scenario_id="platform-42", + sub_goals=["created_rider", "judged_tone"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing'\n" + ) + }, + # "judged_tone" deliberately has no checks/ file -- a JUDGED sub-goal. + ) + scenarios = ss.load_scenarios(tmp_path) + assert len(scenarios) == 1 + scenario = scenarios[0] + assert scenario.scenario_key == "book_a_ride" + assert scenario.scenario_id == "platform-42" + assert [g.name for g in scenario.sub_goals] == ["created_rider", "judged_tone"] + deterministic, judged = scenario.sub_goals + assert deterministic.judged == "" + assert judged.judged != "" # mandatory, non-empty marker (CONTRACT QUESTIONS). + + +def test_load_scenarios_sorts_by_folder_name_like_folder_py(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "bravo", scenario_key="bravo") + _write_scenario(root, "alpha", scenario_key="alpha") + scenarios = ss.load_scenarios(tmp_path) + assert [s.scenario_key for s in scenarios] == ["alpha", "bravo"] + + +def test_load_scenarios_missing_setup_or_ready_defaults_to_empty_text(tmp_path: Path) -> None: + # `folder.py`'s `read_folder` treats a missing setup.py/ready.py as "" -- mirrored here. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_load_scenarios_raises_typed_error_for_missing_scenario_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", write_body=False) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_invalid_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = root / "s1" + folder.mkdir(parents=True) + (folder / "scenario.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_non_object_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", raw_body=None, write_body=False) + (root / "s1" / "scenario.json").write_text(json.dumps(["not", "an", "object"]), encoding="utf-8") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_non_string_sub_goals(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", raw_body={"scenario_key": "s1", "sub_goals": [1, 2]}) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_when_scenarios_directory_absent(tmp_path: Path) -> None: + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_never_silently_skips_a_bad_folder(tmp_path: Path) -> None: + # folder.py's own `read_all` swallows a bad folder and continues -- this module must not: a + # good scenario alongside a broken one still fails the whole load. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "good", scenario_key="good") + _write_scenario(root, "bad", write_body=False) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# Wrapper field mapping -- work item 3. `scenario_key`/`scenario_id` carried VERBATIM, including +# empty (the empty-key fixture: hand-written, since pr63's model backfills empty keys). +# ================================================================================================= + + +def test_wrapper_carries_empty_scenario_key_and_id_verbatim(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", scenario_id="", sub_goals=[]) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.scenario_key == "" + assert scenario.scenario_id == "" + + +def test_wrapper_preserves_sub_goal_document_order(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["z", "a", "m"]) + scenario = ss.load_scenarios(tmp_path)[0] + assert [g.name for g in scenario.sub_goals] == ["z", "a", "m"] + + +def test_judged_sub_goal_check_returns_none_and_judged_is_non_empty(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["needs_judgment"]) + scenario = ss.load_scenarios(tmp_path)[0] + goal = scenario.sub_goals[0] + assert goal.judged != "" + assert goal.check(object(), []) is None # "held" by the shared return-value convention. + + +def test_deterministic_sub_goal_judged_is_empty(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.sub_goals[0].judged == "" + + +# ================================================================================================= +# Compiler -- work item 2. Good/bad, compiled once at load. +# ================================================================================================= + + +def test_compile_empty_source_is_a_no_op_success(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="", ready_code="") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_setup_syntax_error_fails_at_load_as_scenario_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world:\n pass\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_check_syntax_error_fails_at_load_as_scenario_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["broken"], + checks={"broken": "def check(world, calls\n pass\n"}, + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_missing_entry_point_fails_at_load(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="x = 1\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no setup"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-2 (HIGH, p12-review-r1.md) -- an EXISTING but empty/whitespace-only checks/.py must not +# become a vacuously-passing deterministic goal. Absence of the file is what means "judged"; a +# present-but-empty file is malformed, matching `_compile_entry`'s `allow_empty=False` for `check`. +# ================================================================================================= + + +def test_existing_but_empty_check_file_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": ""}) + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +def test_existing_but_whitespace_only_check_file_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": " \n\t\n"} + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +def test_setup_and_ready_still_allow_empty_source_after_the_r1_2_fix(tmp_path: Path) -> None: + # Regression guard: R1-2's `allow_empty=False` is scoped to `check` only -- setup/ready must + # still treat empty/whitespace source as the pre-existing no-op success (`folder.py` parity). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code=" \n", ready_code="") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_mutation_vacuous_empty_check_pass_is_caught(tmp_path: Path) -> None: + # Mutation table (R1-2): simulates deleting `allow_empty=False` from the `check` call site in + # `_load_one`, i.e. reverting to the pre-fix behavior where an existing-but-empty check file + # silently compiled to a no-op "held" callable -- a vacuous deterministic pass. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": ""}) + + # Baseline: the real fix catches it. + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + def _always_allow_empty(source: str, *, label: str, entry: str, allow_empty: bool = True): + # The mutant: `allow_empty` is accepted but ignored -- `check` is treated exactly like + # `setup`/`ready` again, as if the R1-2 fix's `allow_empty=False` call-site edit were + # reverted. A standalone reimplementation of the pre-fix `_compile_entry` body, not a call + # back into `ss._compile_entry` (which IS this mutant while the patch is active). + del allow_empty + if not source.strip(): + return lambda *args: None + code = compile(source, f"<{label}>", "exec") + namespace: dict[str, Any] = {} + exec(code, namespace) # noqa: S102 + function = namespace.get(entry) + if not callable(function): + raise ss.ScenarioDocumentInvalid(f"{label} defines no {entry}()") + return function + + with mock.patch.object(ss, "_compile_entry", _always_allow_empty): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + goal = scenarios[0].sub_goals[0] + assert goal.judged == "" # still classified deterministic... + assert goal.check(object(), []) is None # ...and the mutant's vacuous "held" verdict + + # Restored: the guard is back. + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-3 (MEDIUM, p12-review-r1.md) -- `sub_goals[]` names are used verbatim to build +# `checks/.py`; an absolute or traversal-shaped name must never resolve to a file outside +# `checks/` (bypassing the bundle's own integrity seal) or silently reclassify a deterministic goal +# as judged (a nonexistent path just fails `is_file()`). +# ================================================================================================= + + +def test_absolute_path_subgoal_name_is_rejected(tmp_path: Path) -> None: + outside = tmp_path / "outside.py" + outside.write_text("def check(world, calls):\n return 'should never run'\n", encoding="utf-8") + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[str(outside.with_suffix(""))]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_relative_traversal_subgoal_name_is_rejected(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["../../../../etc/passwd"]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_backslash_subgoal_name_is_rejected(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["a\\b"]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_plain_subgoal_names_are_unaffected_by_the_r1_3_fix(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["holds", "judged_one"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert [g.name for g in scenario.sub_goals] == ["holds", "judged_one"] + + +def test_mutation_subgoal_name_sanitization_reproduces_the_bundle_escape(tmp_path: Path) -> None: + # Mutation table (R1-3): simulates deleting the `_validate_subgoal_name` call in `_load_one` -- + # an absolute-path sub_goal name would once again resolve `check_path` to a file entirely + # outside `checks/` (and outside the sealed bundle), reading and compiling it. + outside = tmp_path / "outside.py" + outside.write_text("def check(world, calls):\n return 'ran from outside the bundle'\n", encoding="utf-8") + root = tmp_path / ss.SCENARIOS_DIRNAME + escaping_name = str(outside.with_suffix("")) + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[escaping_name]) + + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + with mock.patch.object(ss, "_validate_subgoal_name", lambda name, *, folder_name: None): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + goal = scenarios[0].sub_goals[0] + assert goal.check(object(), []) == "ran from outside the bundle" # read from OUTSIDE checks/ + + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-1 (CRITICAL, p12-review-r1.md) -- untyped/`BaseException` failures in the load path must never +# bypass the one `except ScenarioDocumentInvalid` clause in `hosted_entrypoint.py::run_job`: a +# scenario document must not be able to set the guest's own exit code (module-level `sys.exit()`) +# or vanish with zero terminal events (an unreadable file, a non-UTF-8 file). Reproduced here at +# the reader/compiler level, directly against `ss.load_scenarios` -- the e2e proof (through the +# REAL `run_job`, asserting the exact exit code and the terminal event) lives in +# `test_hosted_entrypoint.py` for the two triggers that survive `process_preflight.py`'s own +# byte-for-byte digest re-verification (module-level `sys.exit()`, non-UTF-8 bytes -- both hash +# fine as raw bytes). The unreadable-file (chmod 000) trigger is NOT reproduced through the full +# bundle/preflight pipeline: `process_preflight.py::_verify_digest` (a file outside this task's +# four-edit allowlist) re-opens every listed file with a bare `path.open("rb")`, unguarded, and +# would raise its own untyped `PermissionError` before `scenario_source.build()` is ever reached -- +# an orthogonal, pre-existing gap in a module this task cannot touch. Reproduced here instead, +# directly at the boundary this task owns. +# ================================================================================================= + + +def test_setup_module_level_sys_exit_zero_is_typed_document_invalid(tmp_path: Path) -> None: + # The worst case in the finding: `sys.exit(0)` inside scenario code, unguarded, previously + # meant the GUEST process itself exited 0 -- a "clean terminal that never happened" (§0.6). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(0)\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_module_level_sys_exit_three_is_typed_document_invalid(tmp_path: Path) -> None: + # EXIT_FENCED == 3: previously the guest would exit 3, read by the platform as a fenced/ + # superseded attempt rather than a scenario content defect. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(3)\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_check_module_level_bare_sys_exit_is_typed_document_invalid(tmp_path: Path) -> None: + # Bare `sys.exit()` (no argument) is `SystemExit()`, not `SystemExit(int)` -- still a + # `BaseException`, not an `Exception`; a `checks/.py` file is exactly where a generator + # could emit this by omitting the `if __name__ == "__main__":` guard around `folder.py`'s own + # `_RUNNABLE` tail (R1-4). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["broken"], + checks={"broken": "import sys\nsys.exit()\n"}, + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_unreadable_file_chmod_000_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world):\n pass\n") + setup_path = folder / "setup.py" + setup_path.chmod(0o000) + try: + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + finally: + setup_path.chmod(0o644) # restore so pytest's tmp_path cleanup can remove it + + +def test_setup_non_utf8_bytes_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = _write_scenario(root, "s1", scenario_key="s1") + (folder / "setup.py").write_bytes(b"def setup(world):\n return '\xff\xfe'\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + + +def test_scenario_json_non_utf8_bytes_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = root / "s1" + folder.mkdir(parents=True) + (folder / "scenario.json").write_bytes(b"\xff\xfe not valid utf-8 at all") + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + + +def test_unreadable_scenarios_directory_is_typed_document_invalid_not_an_escape(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + root.mkdir(parents=True) + (root / "s1").mkdir() + (root / "s1" / "scenario.json").write_text(json.dumps({"scenario_key": "s1"}), encoding="utf-8") + root.chmod(0o000) + try: + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot list"): + ss.load_scenarios(tmp_path) + finally: + root.chmod(0o755) # restore so pytest's tmp_path cleanup can remove it + + +def test_unreadable_scenarios_directory_makes_bundle_has_scenarios_false_not_an_escape( + tmp_path: Path, +) -> None: + # `bundle_has_scenarios` sits OUTSIDE `run_job`'s try/except entirely (R1-1) -- an unreadable + # `scenarios/` must report False (falling back to the safe `NotWiredScenarioSource` default), + # never raise, since there is nothing typed to catch it at that call site. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1") + root.chmod(0o000) + try: + assert ss.bundle_has_scenarios(tmp_path) is False + finally: + root.chmod(0o755) + + +def test_mutation_revert_r1_1_containment_reproduces_the_untyped_escapes(tmp_path: Path) -> None: + # Revert-verify-restore: a SCRATCH copy of this module's pre-R1-1-fix content (never a tracked + # file -- this module did not exist as a tracked file before this task either, see the mutation + # section's own DUPLICATION DISCLOSURE below) is imported under a private name and driven + # through the exact same fixtures the tests above use. It reproduces every one of the four + # untyped escapes the fix closes; the real, fixed `ss` module does not. + import importlib.util + import sys as _sys + + prefix_path = Path( + "/private/tmp/claude-501/-Users-khushalsonawat-Desktop-future-agi/" + "12a30b1b-5fe7-4808-ae3f-103ab50c6ebc/scratchpad/p12fix1/scenario_source_prefix.py" + ) + if not prefix_path.is_file(): + pytest.skip("pre-fix scratch copy not present in this environment") + module_name = "_p12_scenario_source_prefix" + spec = importlib.util.spec_from_file_location(module_name, prefix_path) + assert spec is not None and spec.loader is not None + prefix = importlib.util.module_from_spec(spec) + # dataclasses' `from __future__ import annotations` string-annotation resolution looks the + # module up in `sys.modules` by name -- registered (and cleaned up after) purely for that, + # never left behind for anything else to import. + _sys.modules[module_name] = prefix + try: + spec.loader.exec_module(prefix) + + # (a) module-level sys.exit(0) -- pre-fix: raw SystemExit escapes `load_scenarios` itself. + root = tmp_path / "a" / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(0)\n") + with pytest.raises(SystemExit): + prefix.load_scenarios(tmp_path / "a") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "a") + + # (b) non-UTF-8 setup.py -- pre-fix: raw UnicodeDecodeError escapes. + root_b = tmp_path / "b" / ss.SCENARIOS_DIRNAME + folder_b = _write_scenario(root_b, "s1", scenario_key="s1") + (folder_b / "setup.py").write_bytes(b"def setup(world):\n return '\xff\xfe'\n") + with pytest.raises(UnicodeDecodeError): + prefix.load_scenarios(tmp_path / "b") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "b") + + # (c) unreadable setup.py (chmod 000) -- pre-fix: raw PermissionError escapes. + root_c = tmp_path / "c" / ss.SCENARIOS_DIRNAME + folder_c = _write_scenario( + root_c, "s1", scenario_key="s1", setup_code="def setup(world):\n pass\n" + ) + setup_path = folder_c / "setup.py" + setup_path.chmod(0o000) + try: + with pytest.raises(PermissionError): + prefix.load_scenarios(tmp_path / "c") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "c") + finally: + setup_path.chmod(0o644) + finally: + del _sys.modules[module_name] + + +# ================================================================================================= +# Security-shaped (documentation, not enforcement) -- work item 5d. The precedent (`folder.py`'s +# `_run`) restricts nothing: a bare `{}` namespace with full default builtins still reaches `os`, +# `open`, `subprocess`. Pinned here rather than assumed. CONTRACT QUESTIONS: scenario code in the +# hosted guest is unsandboxed beyond the sandbox itself -- no sandbox is added by this module. +# ================================================================================================= + + +def test_scenario_code_is_unsandboxed_importing_os_runs_successfully(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", + setup_code=( + "import os\n" + "def setup(world):\n" + " del world\n" + " return os.getcwd() and None\n" + ), + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None # ran to completion -- `import os` was never blocked. + + +# ================================================================================================= +# Consumer proof (work item 5b) -- real documents -> this adapter -> the REAL HostedScheduler, with +# a fake world/call runner. `WorldPool` needs no real `EnvironmentBundleV2`/provisioner fidelity +# for this (`bundle=object()`, matching test_hosted_scheduler.py's own `_pool` helper) -- only +# `provision()`/`reset()`/`healthy()`/`close()` are ever called on it. +# ================================================================================================= + + +class _FakeProvisioner: + name = "fake-process" + + def __init__(self, instances: int) -> None: + self.instances = instances + self.closed = False + self._runtimes = { + i: EnvironmentRuntime( + runtime_id=f"digest:w{i}", world_index=i, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + for i in range(instances) + } + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + del bundle, source, bundle_dir, work_directory, contract + return [self._runtimes[i] for i in range(instances)] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + del runtime, work_directory + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + del runtime, work_directory + return True + + async def close(self, *, work_directory: Path) -> None: + del work_directory + self.closed = True + + +class _FakeWorld: + """Unlike `test_hosted_scheduler.py`'s `InMemoryWorld`, `read_only()` SHARES the `rows` dict + rather than starting a fresh one -- this consumer-proof test needs a `check()` to see what + `setup()` actually wrote, or the deterministic-check assertion below would pass vacuously.""" + + def __init__(self, world_index: int, rng: Any, rows: dict[str, list[dict[str, Any]]] | None = None) -> None: + self.world_index = world_index + self.rng = rng + self.rows: dict[str, list[dict[str, Any]]] = rows if rows is not None else {} + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return dict(self.rows) if table is None else {table: list(self.rows.get(table, []))} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + self.rows.setdefault(collection, []).append(record) + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + del key, changes, by + return 0 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + del key, by + return 0 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + del sql, params + return [] + + def read_only(self) -> "_FakeWorld": + return _FakeWorld(self.world_index, self.rng, rows=self.rows) + + +class _FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> _FakeWorld: + return _FakeWorld(runtime.world_index, rng) + + +class _FakeCallRunner: + def __init__(self, outcomes: dict[str, CallOutcome | Exception]) -> None: + self._outcomes = outcomes + + async def run(self, scenario: Any, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + outcome = self._outcomes[scenario.scenario_key] + if isinstance(outcome, Exception): + raise outcome + return outcome + + +@dataclass +class _FakeOutbound: + events: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + receipts: list[Any] = field(default_factory=list) + + async def scenario_started(self, *, scenario_key: str, world_index: int, scenario_attempt: int) -> None: + self.events.append(("scenario_started", {"scenario_key": scenario_key})) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + self.events.append(("scenario_retried", {})) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + self.events.append(("world_unhealthy", {"cause": cause})) + + async def log(self, *, level: str, message: str) -> None: + self.events.append(("log", {"message": message})) + + async def receipt(self, receipt: Any) -> None: + self.receipts.append(receipt) + + +def _call_outcome() -> CallOutcome: + return CallOutcome( + calls=(Call(name="tool", arguments={}, result="ok", ok=True, error="", refused=False, at=0.0),), + turns=1, started_at="2026-08-25T00:00:00.000Z", ended_at="2026-08-25T00:00:01.000Z", + duration_ms=1000, + ) + + +def _build_fixture_bundle(tmp_path: Path) -> Path: + """Two scenarios: one whose deterministic check genuinely holds against what `setup` wrote, + one whose deterministic check genuinely does not -- proving verdicts are evaluated for real, + not just that a key made it through (the brief's own "key-only assertion cannot detect a + wrapper bug" warning). Each also carries one judged sub-goal.""" + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "passing", + scenario_key="passing", scenario_id="", + sub_goals=["created_rider", "needs_judgment"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ) + }, + ) + _write_scenario( + root, "failing", + scenario_key="failing", scenario_id="", + sub_goals=["created_rider"], + setup_code="def setup(world):\n return None\n", # never creates the rider + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ) + }, + ) + return tmp_path + + +def test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() -> None: + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-consumer-")) + bundle_dir = _build_fixture_bundle(tmp_path) + scenarios = ss.load_scenarios(bundle_dir) + assert [s.scenario_key for s in scenarios] == ["failing", "passing"] # sorted by folder name + + outbound = _FakeOutbound() + provisioner = _FakeProvisioner(1) + pool = WorldPool( + provisioner, bundle=object(), source=Path("/work/source"), bundle_dir=bundle_dir, + work_directory=tmp_path, instances=1, outbound=outbound, + ) + await pool.start() + call_runner = _FakeCallRunner( + {"passing": _call_outcome(), "failing": _call_outcome()} + ) + scheduler = HostedScheduler( + pool=pool, world_factory=_FakeWorldFactory(), call_runner=call_runner, + outbound=outbound, job_seed=1, + ) + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=10.0) + await pool.close() + + assert result.aborted is None + receipts_by_key = {r.scenario_key: r for r in result.receipts} + assert set(receipts_by_key) == {"passing", "failing"} + + passing = receipts_by_key["passing"] + assert passing.status == "passed" + goals_by_name = {g.name: g for g in passing.sub_goals} + assert goals_by_name["created_rider"].held is True # a real deterministic check, evaluated + assert goals_by_name["created_rider"].judged is False + assert goals_by_name["needs_judgment"].held is True # placeholder "held" convention + assert goals_by_name["needs_judgment"].judged is True # marks it as not really settled yet + + failing = receipts_by_key["failing"] + assert failing.status == "failed" # the SAME check, genuinely evaluated, genuinely fails + assert failing.sub_goals[0].held is False + assert failing.sub_goals[0].reason == "missing rider" + + asyncio.run(scenario()) + + +def test_runtime_error_inside_setup_reaches_setup_crashed_through_the_real_scheduler() -> None: + # Divergence (a) from folder.py's `_run`: a syntax error is caught at LOAD time. A RUNTIME + # error inside a setup() that compiles fine is a different story -- it must reach + # `hosted_scheduler.py`'s own classification (`_run_phase`'s `_PhaseCrashed` -> + # `setup_crashed`), proven here through the real scheduler, not a fake standing in for it. + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-setup-crash-")) + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "boom", scenario_key="boom", scenario_id="", + sub_goals=[], + setup_code="def setup(world):\n raise ValueError('setup blew up')\n", + ) + scenarios = ss.load_scenarios(tmp_path) + + outbound = _FakeOutbound() + provisioner = _FakeProvisioner(1) + pool = WorldPool( + provisioner, bundle=object(), source=Path("/work/source"), bundle_dir=tmp_path, + work_directory=tmp_path, instances=1, outbound=outbound, + ) + await pool.start() + scheduler = HostedScheduler( + pool=pool, world_factory=_FakeWorldFactory(), call_runner=_FakeCallRunner({}), + outbound=outbound, job_seed=1, + ) + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=10.0) + await pool.close() + + assert len(result.receipts) == 1 + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None + assert receipt.failure.code == "setup_crashed" + assert "setup blew up" in receipt.failure.message + + asyncio.run(scenario()) + + +# ================================================================================================= +# R1-4 (MEDIUM, p12-review-r1.md) -- nothing else in this suite feeds the adapter an actual +# `folder.py::write_folder` product; every other fixture here is hand-written. This pins the reader +# against the REAL producer once: a future change to `write_folder`'s layout (e.g. setting +# `namespace["__name__"]` inside `_compile_entry` to make tracebacks readable) would turn every real +# check file's `_RUNNABLE` tail (which every one of them carries) into a module-level `SystemExit` +# at load -- exactly R1-1's worst case -- and an all-hand-written suite would never see it. +# ================================================================================================= + + +def test_real_write_folder_round_trip_matches_the_adapters_reading(tmp_path: Path) -> None: + from fi.alk.harness import folder as fmod + from fi.alk.harness.catalogue import Catalogue, SubGoal + from fi.alk.harness.scenario import Scenario + + catalogue = Catalogue( + sub_goals=[ + SubGoal( + name="created_rider", what="rider row exists", + check=( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ), + ), + SubGoal(name="polite_tone", what="agent was polite", judged="was the refusal explained?"), + ] + ) + scenario_model = Scenario( + name="book_a_ride", + instruction="book a ride", + sub_goals=["created_rider", "polite_tone"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + ) + fmod.write_folder(scenario_model, catalogue, tmp_path) + + # The real producer's own on-disk layout must satisfy `bundle_has_scenarios`'s presence test + # unchanged. + assert ss.bundle_has_scenarios(tmp_path) is True + + scenarios = ss.load_scenarios(tmp_path) + assert len(scenarios) == 1 + scenario = scenarios[0] + goals_by_name = {g.name: g for g in scenario.sub_goals} + assert set(goals_by_name) == {"created_rider", "polite_tone"} + assert goals_by_name["created_rider"].judged == "" # has a real checks/ file + assert goals_by_name["polite_tone"].judged != "" # no checks/ file -- judged, per deterministic() + + world = _FakeWorld(0, random.Random(0)) + assert scenario.setup(world) is None + assert world.rows == {"riders": [{"id": 1}]} + assert scenario.ready(world) is None + # The real producer appends `_RUNNABLE` (ending `if __name__ == "__main__": ... raise + # SystemExit(...)`) to every checks/ file it writes -- this proves `exec(code, {})` resolves + # `__name__` to `'builtins'` (never `'__main__'`), so that tail stays inert, against the REAL + # producer's own bytes rather than a hand-written stand-in that never carries the tail at all. + assert goals_by_name["created_rider"].check(world, []) is None # held, genuinely evaluated + assert goals_by_name["polite_tone"].check(world, []) is None # judged placeholder convention + + +# ================================================================================================= +# R1-5 (MEDIUM, p12-review-r1.md) -- divergence (a) (compile once, at load) moves every scenario +# file's compile+exec OUTSIDE every phase budget the scheduler enforces. A wall-clock budget around +# the load converts a hang into a typed terminal instead of an unbounded one (a worker thread +# cannot actually be canceled, so the background thread is accepted as leaked -- see +# `_LOAD_TIMEOUT_SECONDS`'s docstring). +# ================================================================================================= + + +def test_load_timeout_converts_a_hanging_module_level_scenario_into_a_typed_failure() -> None: + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-load-timeout-")) + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", + # Module-level, not inside setup() -- runs during `_compile_entry`'s `exec`, i.e. + # during the load itself, which is exactly what a real budget must bound. + setup_code="import time\ntime.sleep(1.5)\ndef setup(world):\n pass\n", + ) + source = ss.BundleScenarioSource() + with mock.patch.object(ss, "_LOAD_TIMEOUT_SECONDS", 0.1): + with pytest.raises(ss.ScenarioDocumentInvalid, match="exceeded"): + await source.build( + object(), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) + + asyncio.run(scenario()) + + +def test_load_without_a_hang_is_unaffected_by_the_budget(tmp_path: Path) -> None: + async def scenario() -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[]) + source = ss.BundleScenarioSource() + scenarios = await source.build( + object(), object(), object(), pool=object(), world_factory=object(), bundle_dir=tmp_path + ) + assert [s.scenario_key for s in scenarios] == ["s1"] + + asyncio.run(scenario()) + + +# ================================================================================================= +# Mutations (work item 6) -- proved via in-memory monkeypatching (`unittest.mock.patch.object`) +# rather than editing bytes on any tracked file on disk: this module (`scenario_source.py`) is a +# brand-new, untracked file at the start of this work, so there is no tracked-file shasum to record +# for these -- the git-recovery safety net the brief's mutation section is guarding against +# (touching a TRACKED file's bytes with no git-based way back) does not apply to a file this task +# itself created. Each mutation patches one function/attribute, runs the assertion that should now +# fail (or the behavior that should now differ), and restores in a `finally` -- `mock.patch.object` +# does this atomically and cannot leave the module in a mutated state even if the assertion raises. +# ================================================================================================= + + +def test_mutation_skip_compile_check_is_killed(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world:\n pass\n") + + # Baseline: the real compiler catches the syntax error. + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + # Mutant: a "compiler" that never raises on a bad compile, returning a no-op instead -- + # simulates deleting the try/except around `compile()`/`exec()` in `_compile_entry`. + def _never_fails(source: str, *, label: str, entry: str): + del source, label, entry + return lambda *args: None + + with mock.patch.object(ss, "_compile_entry", _never_fails): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + assert scenarios[0].setup(object()) is None # confirms the mutant path actually ran + + # Restored: the guard is back. + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_mutation_judged_flag_dropped_is_killed_by_5b_verdict_assertions() -> None: + # Mutant: `_load_one` always sets `judged=""`, as if the "no checks/.py -> judged" + # branch were deleted. + # + # R1-6/R1-8 fold-in (p12-review-r1.md LOW findings): the original version of this test named + # the 5b consumer-proof test in its own identifier but never actually ran it -- it asserted on + # `_load_one`'s output directly instead, which is true but does not exercise the killing + # assertion it claims to. This version runs the REAL + # `test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios` under the patch and checks + # THAT it raises. No `async def`/`asyncio.run` wrapper needed either (R1-8): that test already + # drives its own `asyncio.run` internally, and nothing here awaits anything. + original_load_one = ss._load_one + + def _dropped_judged(folder: Path): + compiled = original_load_one(folder) + broken_goals = tuple( + ss._CompiledSubGoal(name=g.name, judged="", check=g.check) for g in compiled.sub_goals + ) + return ss._CompiledScenario( + scenario_key=compiled.scenario_key, scenario_id=compiled.scenario_id, + sub_goals=broken_goals, setup=compiled.setup, ready=compiled.ready, + ) + + with mock.patch.object(ss, "_load_one", _dropped_judged): + with pytest.raises(AssertionError): + test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() + + # Restored: the real 5b test passes again. + test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() + + +def test_mutation_empty_key_reader_synthesizing_a_key_is_caught(tmp_path: Path) -> None: + # Mutant: the reader backfills an empty `scenario_key` with a placeholder instead of carrying + # it verbatim, as pr63's own model validator would. The empty-key fixture's own guarantee + # (verbatim empty carry) must catch this. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", scenario_id="", sub_goals=[]) + + real = ss.load_scenarios(tmp_path)[0] + assert real.scenario_key == "" # baseline: verbatim carry holds + + original_load_one = ss._load_one + + def _synthesizes_key(folder: Path): + compiled = original_load_one(folder) + key = compiled.scenario_key or "synthesized-key" + return ss._CompiledScenario( + scenario_key=key, scenario_id=compiled.scenario_id, sub_goals=compiled.sub_goals, + setup=compiled.setup, ready=compiled.ready, + ) + + with mock.patch.object(ss, "_load_one", _synthesizes_key): + mutant = ss.load_scenarios(tmp_path)[0] + assert mutant.scenario_key != "" # the mutant's defect: no longer verbatim + + restored = ss.load_scenarios(tmp_path)[0] + assert restored.scenario_key == "" # confirms the patch was fully undone From 89f1ce2f37dfe2219e6556c1f859eb11716fc7d0 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 26 Aug 2026 12:10:29 +0530 Subject: [PATCH 16/20] =?UTF-8?q?feat(harness):=20register=20scenarios=20w?= =?UTF-8?q?ith=20the=20platform=20=E2=80=94=20receipts=20now=20deliver?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire register_with_platform() into BundleScenarioSource.build(): one provision + one begin call on the attempt's scenarios endpoint (single POST, body-level operation field — the route the platform actually serves), platform-assigned ids matched to documents BY scenario_key with hard guards (missing/unknown/duplicate key -> typed failure, never a partial assignment), and the full key set sent on begin. Scenario result receipts now carry the platform-assigned scenario_id, so they deliver instead of being dropped for an empty id. The P12 pinning test is split honestly: a delivery proof plus a drop-guard preservation test for the one path that can still see an unregistered scenario. Also guards empty scenario_key before the network call, keeping that deterministic content defect in the environment domain instead of letting the platform's 400 reclassify it as platform_sync. ScenariosClient path defaults collapse to "" (provision/begin suffixes would 404 against the real router). Cold review: CLEARS — 7/7 mutations killed, 823 passed / 0 failed x2. Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_entrypoint.py | 26 +- src/fi/alk/harness/scenario_source.py | 190 ++++++++++-- tests/harness/test_hosted_entrypoint.py | 200 +++++++++++-- tests/harness/test_scenario_source.py | 379 +++++++++++++++++++++++- 4 files changed, 743 insertions(+), 52 deletions(-) diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index abc1ee3c..0d0eb77e 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -406,12 +406,22 @@ def __init__(self, error: ob.ChannelError | None) -> None: class ScenariosClient: - """CROSS-DOC GAP: the Scenario Generation Contract that defines the exact - `provision`/`begin` payload and path shape is Karthik's, "in review," and not available to this - module. `provision_path`/`begin_path` are constructor-injectable placeholders rather than a - guess baked into the URL, so the real paths can be supplied without touching this class once - that contract lands. Shares `channel_state` with the other three channels (a fence on any one - must stop all of them, per outbound.py's own `ChannelState` docstring).""" + """RESOLVED (p13-worker-r2, reports/p13-worker-r2.md CONTRACT NOTES): Karthik's Scenario + Generation Contract (PR #63) documented two paths (`run-tests/provision/` + + `run-tests/{id}/test-executions/`) and a position-ordered `scenario_ids` response, but the + platform's actual, live route (futureagi/simulate/views/hosted_harness.py:78-90, + urls.py:128-132) mints exactly ONE url per attempt -- a DRF detail `@action` with no + `url_path`, so the router only ever produces `.../scenarios/`, never a `provision/`/`begin/` + sub-resource. The real dispatch key is a body-level `operation: "provision"|"begin"` field + (serializers/hosted_harness.py:201-226's `HarnessScenarioOperationSerializer`). This class's + transport (`_post`) is unchanged -- `provision_path`/`begin_path` are the SAME + constructor-injectable placeholders as before, now correctly defaulted to an EMPTY suffix (the + real route needs none) rather than a guessed path segment; `register_with_platform` + (scenario_source.py) is what adds the `operation` field into each payload before calling + `.provision()`/`.begin()`, matching this class's existing "operation field in payload" seam + rather than requiring a change to either method's body. Shares `channel_state` with the other + three channels (a fence on any one must stop all of them, per outbound.py's own `ChannelState` + docstring).""" def __init__( self, @@ -422,8 +432,8 @@ def __init__( sleep: Callable[[float], None] = time.sleep, rng: Callable[[], float] = random.random, channel_state: ob.ChannelState | None = None, - provision_path: str = "provision/", - begin_path: str = "begin/", + provision_path: str = "", + begin_path: str = "", ) -> None: self._capabilities = capabilities self._transport = transport or ob.RequestsTransport() diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py index 3a4192b5..80455f40 100644 --- a/src/fi/alk/harness/scenario_source.py +++ b/src/fi/alk/harness/scenario_source.py @@ -10,19 +10,26 @@ instead of depending on either model -- see the report's design-decisions section for the consequences of that choice (HEAD-model drift). -Karthik's Scenario Generation Contract (the `provision`/`begin` wire shapes) has not landed. This -module builds the bundle-reading + compiling + wrapping side only; `register_with_platform` below -is the one seam a later change wires in once that contract exists. +RESOLVED (p13-worker-r2, reports/p13-worker-r2.md CONTRACT NOTES): the `provision`/`begin` wire +shapes below follow the platform's actual, live route (futureagi/simulate/serializers/services/ +views `hosted_harness.py`) rather than Karthik's Scenario Generation Contract text (PR #63), where +the two disagree -- a single `POST .../scenarios/` discriminated by a body-level `operation` field, +`begin` keyed on the full `scenario_keys` set, and a provision response KEYED by `scenario_key` +(never a position-ordered array). `register_with_platform` below is the seam that builds those +payloads and merges the platform-assigned `scenario_id`s back onto each scenario. """ from __future__ import annotations import asyncio import json -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Sequence +from . import outbound as ob +from .job import FailureDomain + if TYPE_CHECKING: from .hosted_entrypoint import ScenariosClient @@ -323,12 +330,12 @@ async def build( world_factory: Any, bundle_dir: Path, ) -> Sequence[_CompiledScenario]: - del job, bundle, scenarios_client, pool, world_factory + del bundle, pool, world_factory # `Path.read_text`/`iterdir`/`compile` are all blocking filesystem+CPU work -- run off the # event loop the same way `hosted_entrypoint.py` already does for `bundle_source.load` and # `preflight_bundle`, rather than stalling every other in-flight scenario behind it. try: - return await asyncio.wait_for( + scenarios = await asyncio.wait_for( asyncio.to_thread(load_scenarios, bundle_dir), timeout=_LOAD_TIMEOUT_SECONDS ) except asyncio.TimeoutError as exc: @@ -340,22 +347,167 @@ async def build( f"{bundle_dir / SCENARIOS_DIRNAME}: loading scenario documents exceeded " f"{_LOAD_TIMEOUT_SECONDS:.0f}s" ) from exc + # An empty `scenario_key` is carried VERBATIM off the document by design (module docstring + # -- never synthesized here), but it is also the one shape `hosted_entrypoint.py`'s own + # downstream `_validate_scenarios` would reject as a local, deterministic ENVIRONMENT-domain + # content defect -- checked HERE, before `register_with_platform` ever reaches the network, + # so that cheaper, existing local classification wins over a round trip that would only + # rediscover the same defect as a `platform_sync` failure instead (Azain's serializer + # rejects a blank `scenario_key` with its own 400 -- `scenario_key` is a plain + # non-`allow_blank` `CharField`, hosted_harness.py:169). `ScenarioDocumentInvalid` reuses + # `run_job`'s EXISTING `except ScenarioDocumentInvalid` clause (domain=environment) -- + # nothing new to catch there. + if any(not scenario.scenario_key for scenario in scenarios): + raise ScenarioDocumentInvalid( + f"{bundle_dir / SCENARIOS_DIRNAME}: a scenario document has no non-empty " + "scenario_key" + ) + # p13: pre-allocation, after load and before the scheduler ever sees a scenario (spine + # step 3.5) -- `register_with_platform` raises `ScenarioPreallocationError`/ + # `ob.HostedFencedError`/`ob.HostedChannelFailedError`/`ob.HostedAttemptSupersededError` on + # any failure, all of which `hosted_entrypoint.run_job`'s existing call site around + # `scenario_source.build()` already maps to the typed `validating_scenarios`/`platform_sync` + # terminal (or the fenced exit) -- nothing new to catch here. + return await register_with_platform(scenarios_client, scenarios, run_name=job.run_id) + + +def _preallocation_error(code: str, message: str) -> Exception: + """Builds a `hosted_entrypoint.ScenarioPreallocationError` for a guard failure below -- + imported lazily (not at module level) because `hosted_entrypoint.py` imports THIS module at + its own top level (`BundleScenarioSource`/`ScenarioDocumentInvalid`/`bundle_has_scenarios`), so + a top-level import back would be a circular import. Reusing that exact exception class (rather + than inventing a new one) is what lets these guard failures land on `run_job`'s ALREADY-WIRED + `except (ScenarioSourceNotWired, ScenarioPreallocationError)` clause with no changes there. + """ + from .hosted_entrypoint import ScenarioPreallocationError + + return ScenarioPreallocationError( + ob.ChannelError(ob.ChannelOutcome.PERMANENT_ITEM, FailureDomain.PLATFORM_SYNC, code, message) + ) + + +def _provision_payload(run_name: str, scenarios: Sequence[_CompiledScenario]) -> dict[str, Any]: + """`HarnessScenarioProvisionSerializer`/`HarnessProvisionPersonaSerializer` + (futureagi/simulate/serializers/hosted_harness.py:168-190): `operation`/`name`/`personas` (with + each persona's `scenario_key`) are the only fields this module can actually supply -- `name`/ + `role`/`situation`/`outcome`/`persona` are all `required=False` on the real serializer, and + `_CompiledScenario` itself carries none of them BY DESIGN (this module's own LAYOUT DECISION, + see the module docstring: it reads `scenario.json` for the scheduler-facing fields only, never + through pr63's full `Scenario` model). Sending bare `scenario_key` per persona still validates + against the real endpoint; see CONTRACT NOTES in reports/p13-worker-r2.md. + """ + return { + "operation": "provision", + "name": run_name, + "personas": [{"scenario_key": scenario.scenario_key} for scenario in scenarios], + } + + +def _begin_payload(run_test_id: str, scenarios: Sequence[_CompiledScenario]) -> dict[str, Any]: + """`HarnessScenarioBeginSerializer` (futureagi/simulate/serializers/hosted_harness.py:193-198): + `scenario_keys` is `allow_empty=False` and REQUIRED, and `begin_scenarios` + (services/hosted_harness.py:323-329) 409s (`scenario_key_mismatch`) on anything but an EXACT + match against the full sealed set -- there is no "subset to run" semantics on the real + platform (Karthik's contract text describes an optional partial-subset `scenario_ids`; the + live route does not implement that -- CONTRACT NOTES). The full set is sent every time. + """ + return { + "operation": "begin", + "run_test_id": run_test_id, + "scenario_keys": [scenario.scenario_key for scenario in scenarios], + } + + +def _scenario_ids_by_key( + submitted: Sequence[_CompiledScenario], raw_scenarios: Any +) -> dict[str, str]: + """Matches the platform's KEYED provision response + (`{"scenarios": [{"scenario_key", "scenario_id"}, ...]}`, + futureagi/simulate/serializers/hosted_harness.py:251-260 + + services/hosted_harness.py:487-501's `_provision_response`) back onto `submitted` BY + `scenario_key` -- a dict lookup, never a positional zip. A positional zip (matching Karthik's + documented `scenario_ids` array shape, not what the platform actually returns) would silently + mismatch scenario_id -> scenario the instant the response order differs from `submitted`'s + order, which nothing on the wire guarantees. Every check below raises rather than returning a + partial mapping -- "never partial assignment" per the brief: the caller only gets a mapping + once it is proven complete (every submitted key present, exactly once) and exact (no + unrecognized key). + """ + if not isinstance(raw_scenarios, list): + raise _preallocation_error( + "scenarios_provision_response_invalid", "response 'scenarios' is not a list" + ) + by_key: dict[str, str] = {} + for entry in raw_scenarios: + if not isinstance(entry, dict): + raise _preallocation_error( + "scenarios_provision_response_invalid", "a 'scenarios' entry is not an object" + ) + key = entry.get("scenario_key") + scenario_id = entry.get("scenario_id") + if not isinstance(key, str) or not key: + raise _preallocation_error( + "scenarios_provision_response_invalid", + "a 'scenarios' entry has no non-empty scenario_key", + ) + if key in by_key: + raise _preallocation_error( + "scenario_registration_duplicate_key", + f"scenario_key {key!r} appears more than once in the provision response", + ) + if not isinstance(scenario_id, str) or not scenario_id: + raise _preallocation_error( + "scenarios_provision_response_invalid", + f"scenario_key {key!r} has no non-empty scenario_id", + ) + by_key[key] = scenario_id + + submitted_keys = [scenario.scenario_key for scenario in submitted] + unknown = sorted(set(by_key) - set(submitted_keys)) + if unknown: + raise _preallocation_error( + "scenario_registration_unknown_key", + f"provision response named scenario_key(s) never submitted: {unknown}", + ) + missing = sorted(set(submitted_keys) - set(by_key)) + if missing: + raise _preallocation_error( + "scenario_registration_missing", + f"provision response is missing scenario_key(s): {missing}", + ) + return by_key async def register_with_platform( - scenarios_client: "ScenariosClient", scenarios: Sequence[_CompiledScenario] + scenarios_client: "ScenariosClient", + scenarios: Sequence[_CompiledScenario], + *, + run_name: str, ) -> Sequence[_CompiledScenario]: - """SEAM -- not called anywhere in this module, and not wired into `BundleScenarioSource.build` - above. Once scenario-generation-contract.md section 3 publishes the `provision`/`begin` payload - and response shapes, this is where they get built from `scenarios` and posted through - `scenarios_client.provision(...)`/`.begin(...)`, and where the platform-assigned `scenario_id`s - that `provision` returns get merged back onto each scenario before `build` hands the list to - the scheduler. Left unimplemented rather than guessing a body the contract has not published - (CONTRACT GAP) -- wiring this in is the one remaining integration step this module cannot - finish alone. + """The scenario pre-allocation SEAM, now wired against the platform's real route (a single + `POST .../scenarios/`, discriminated by a body-level `operation` field -- see + `ScenariosClient`'s own docstring for the file:line evidence). `.provision()`/`.begin()` are + blocking network calls (same `ScenariosClient` the rest of `hosted_entrypoint.py` already + drives off the event loop via `asyncio.to_thread` -- matched here rather than diverging). + + Sequence: provision (get platform-assigned ids, keyed by `scenario_key`) -> match ids back + onto `scenarios` with hard guards (`_scenario_ids_by_key`, raises before ANY assignment on any + mismatch) -> begin (seals execution against the FULL scenario_keys set; a begin failure means + NO scenario in this batch is returned with an id -- the whole call raises, same as a provision + failure) -> only then build and return the new scenario list with `scenario_id` filled in. """ - del scenarios_client, scenarios - raise NotImplementedError( - "register_with_platform: scenario-generation-contract.md section 3 (the provision/begin " - "wire shapes) has not landed -- this seam is intentionally left unwired" + provision_result = await asyncio.to_thread( + scenarios_client.provision, _provision_payload(run_name, scenarios) + ) + run_test_id = provision_result.get("run_test_id") + if not isinstance(run_test_id, str) or not run_test_id: + raise _preallocation_error( + "scenarios_provision_response_invalid", "provision response has no run_test_id" + ) + id_by_key = _scenario_ids_by_key(scenarios, provision_result.get("scenarios")) + + await asyncio.to_thread(scenarios_client.begin, _begin_payload(run_test_id, scenarios)) + + return tuple( + replace(scenario, scenario_id=id_by_key[scenario.scenario_key]) for scenario in scenarios ) diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 450470e0..1de92836 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -266,10 +266,23 @@ def request( self.artifacts[digest] = bytes(payload) return ob.TransportResponse(200 if existed else 201, {}, {}) if "/scenarios/" in url and method == "POST" and json_body is not None: + # p13: Azain's real router mints exactly ONE url per attempt (a DRF detail `@action`, + # no `url_path`) -- provision vs begin is a body-level `operation` field, never a URL + # suffix, so routing here is on `json_body["operation"]`, not `url`. self.scenarios_calls.append((url, json_body)) - if url.endswith("/provision/"): - ids = {key: f"platform-{key}" for key in json_body.get("scenario_keys", [])} - return ob.TransportResponse(200, {"result": {"scenario_ids": ids}}, {}) + operation = json_body.get("operation") + if operation == "provision": + keys = [p.get("scenario_key") for p in json_body.get("personas", [])] + scenarios = [{"scenario_key": key, "scenario_id": f"platform-{key}"} for key in keys] + return ob.TransportResponse( + 200, {"result": {"run_test_id": "run-test-1", "scenarios": scenarios}}, {} + ) + if operation == "begin": + return ob.TransportResponse( + 200, {"result": {"test_execution_id": "exec-1", "scenarios": []}}, {} + ) + # No/unknown `operation` -- exercised by callers (e.g. `FakeScenarioSource`) that only + # care about the call happening, never the response shape. return ob.TransportResponse(200, {"result": {"ok": True}}, {}) return ob.TransportResponse( 404, {"error": "not_found", "message": f"unmapped route: {url}", "retryable": False}, {} @@ -472,11 +485,18 @@ async def build( # fake -- accepted only because `run_job` now forwards it to every `ScenarioSource.build`, # injected or not. del job, bundle, pool, world_factory, bundle_dir + # `operation` mirrors what `register_with_platform` (scenario_source.py) really sends -- + # this fake's own `scenario_keys`/`scenario_ids` bodies are otherwise arbitrary (never + # parsed by `FakeTransport`, which only inspects `operation` to pick a response), kept only + # so `test_scenarios_channel_uses_bearer_auth_never_api_key` and friends see two distinct, + # non-empty POST bodies. await asyncio.to_thread( scenarios_client.provision, - {"scenario_keys": [s.scenario_key for s in self._scenarios]}, + {"operation": "provision", "scenario_keys": [s.scenario_key for s in self._scenarios]}, + ) + await asyncio.to_thread( + scenarios_client.begin, {"operation": "begin", "scenario_ids": {}} ) - await asyncio.to_thread(scenarios_client.begin, {"scenario_ids": {}}) return self._scenarios @@ -718,11 +738,40 @@ async def scenario() -> None: def test_scenarios_client_provision_unwraps_the_result_envelope() -> None: + # p13: keyed response (`{"scenarios": [{"scenario_key", "scenario_id"}, ...]}`), matching the + # real platform's `_provision_response` (services/hosted_harness.py:487-501) -- never a + # position-ordered `scenario_ids` array. + capabilities = _capabilities() + transport = FakeTransport() + client = he.ScenariosClient(capabilities, transport) + result = client.provision( + { + "operation": "provision", "name": "run-1", + "personas": [{"scenario_key": "a"}, {"scenario_key": "b"}], + } + ) + assert result == { + "run_test_id": "run-test-1", + "scenarios": [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "b", "scenario_id": "platform-b"}, + ], + } + + +def test_scenarios_client_provision_and_begin_hit_the_same_single_url() -> None: + # p13: Azain's router mints exactly ONE url per attempt (views/hosted_harness.py:78-90's + # `scenarios` detail `@action`, no `url_path`; urls.py:128-132) -- `provision_path`/ + # `begin_path` default to an EMPTY suffix now (not the old guessed `"provision/"`/`"begin/"`, + # which 404 against the real router), so both calls land on `capabilities.endpoints.scenarios` + # itself, discriminated only by the body's `operation` field. capabilities = _capabilities() transport = FakeTransport() client = he.ScenariosClient(capabilities, transport) - result = client.provision({"scenario_keys": ["a", "b"]}) - assert result == {"scenario_ids": {"a": "platform-a", "b": "platform-b"}} + client.provision({"operation": "provision", "name": "run-1", "personas": []}) + client.begin({"operation": "begin", "run_test_id": "run-test-1", "scenario_keys": []}) + urls = [call["url"] for call in transport.calls] + assert urls == [capabilities.endpoints.scenarios, capabilities.endpoints.scenarios] def test_scenarios_client_fencing_latches_the_shared_channel_state() -> None: @@ -731,7 +780,7 @@ def test_scenarios_client_fencing_latches_the_shared_channel_state() -> None: channel_state = ob.ChannelState() client = he.ScenariosClient(capabilities, transport, channel_state=channel_state) try: - client.provision({"scenario_keys": []}) + client.provision({"operation": "provision", "name": "run-1", "personas": []}) except ob.HostedFencedError: pass else: @@ -1246,9 +1295,15 @@ async def scenario() -> None: assert harness.provisioner.provision_calls >= 1 assert harness.provisioner.closed is True - # Scenario pre-allocation (item 3) actually ran against endpoints.scenarios. - assert any(url.endswith("/provision/") for url, _ in harness.transport.scenarios_calls) - assert any(url.endswith("/begin/") for url, _ in harness.transport.scenarios_calls) + # Scenario pre-allocation (item 3) actually ran against endpoints.scenarios -- p13: a + # single url, discriminated by the body's `operation` field (never a `/provision/`/ + # `/begin/` url suffix, which 404s against the real platform router). + assert any( + body.get("operation") == "provision" for _, body in harness.transport.scenarios_calls + ) + assert any( + body.get("operation") == "begin" for _, body in harness.transport.scenarios_calls + ) asyncio.run(scenario()) @@ -2363,16 +2418,17 @@ def test_mutation_adapter_off_makes_the_e2e_test_fail() -> None: test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() -def test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema() -> None: - # NEWLY DISCOVERED while writing the test above: `outbound.py`'s `ResultReceiptDraft` schema - # requires `scenario_id` to be non-empty (pydantic `min_length=1`). The brief mandates carrying - # `scenario_id` VERBATIM off the document -- including empty, until pre-allocation is wired -- - # so a scenario whose pre-allocation has not run gets its receipt rejected at construction, - # logged as an error, and DROPPED, while the job still reports COMPLETED with that scenario - # counted as `passed`/`failed` in `scenario_counts`. This sharpens the brief's own "blocking - # integration obligation" from a documentation concern into a concrete, verified one: today, a - # bundle-sourced scenario can never actually deliver a receipt to the platform until pre- - # allocation assigns it a real `scenario_id`. See CONTRACT QUESTIONS in the report. +def test_bundle_scenario_id_is_assigned_by_registration_and_receipt_now_delivers() -> None: + # p13 UPDATE of the former `test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema` + # (p12): that test pinned a real gap -- `outbound.py`'s `ResultReceiptDraft` schema requires + # `scenario_id` non-empty (pydantic `min_length=1`), and before this task nothing ever filled + # it in, so a bundle-sourced scenario's receipt was silently dropped. Registration + # (`register_with_platform`, scenario_source.py) now runs between load and the scheduler and + # OVERWRITES `scenario_id` with the platform-assigned one before `BundleScenarioSource.build` + # ever returns -- the document is written with `scenario_id=""` here specifically to prove the + # id on the wire came from the (fake) platform's provision response, not the document. See + # `test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely` just below for + # the property this test used to pin, preserved on the path that never registers at all. async def scenario() -> None: harness = _build_harness( scenarios=[], instances=1, use_default_scenario_source=True, @@ -2387,6 +2443,47 @@ async def scenario() -> None: code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["scenario_counts"]["passed"] == 1 + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed"} # the receipt DELIVERS now -- no drop. + + (_, body), = [ + (key, body) for key, body in harness.transport.receipts.items() if key[1] == "passing" + ] + # `FakeTransport`'s fake platform assigns `f"platform-{scenario_key}"` -- confirms the id + # on the wire is the PLATFORM's, not the document's own (empty) one. + assert body["scenario_id"] == "platform-passing" + + error_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and record["payload"].get("level") == "error" + and "ResultReceiptDraft" in record["payload"].get("message", "") + ] + assert error_logs == [] # no drop, so no drop log either. + + asyncio.run(scenario()) + + +def test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely() -> None: + # Preserves the property the pre-p13 pinning test proved: a scenario that reaches the + # scheduler with an empty `scenario_id` (never pre-allocated) still has its receipt rejected by + # `ResultReceiptDraft`'s own schema (`min_length=1`) and DROPPED, loudly, rather than crashing + # the job or silently delivering a receipt the platform would 422 anyway. Through + # `BundleScenarioSource` this is now unreachable (registration always assigns a real id or the + # job fails first) -- so this is exercised through `FakeScenarioSource`'s injected-scenario + # path instead, which never calls `register_with_platform` at all (an "unregistered" scenario + # source, same shape as a future ScenarioSource that also skips pre-allocation). + async def scenario() -> None: + scenarios = [FakeScenario("passing", "", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) # FakeScenarioSource, as usual + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() assert len(terminals) == 1 payload = terminals[0]["payload"] @@ -2406,6 +2503,62 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_registration_response_mismatch_reaches_the_typed_platform_sync_terminal() -> None: + # p13: a provision response that fails `_scenario_ids_by_key`'s guards (scenario_source.py -- + # here, naming NO scenario_key at all, so every submitted one is "missing") must fail the + # whole job through the SAME typed `validating_scenarios`/`platform_sync` terminal + # `run_job`'s existing `except (ScenarioSourceNotWired, ScenarioPreallocationError)` clause + # already produces for every other pre-allocation failure -- no new except clause needed. The + # scheduler must never run: no receipt for the scenario ever reaches the platform. + async def scenario() -> None: + class MismatchedProvisionTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if ( + method == "POST" and "/scenarios/" in url and json_body is not None + and json_body.get("operation") == "provision" + ): + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + self.scenarios_calls.append((url, json_body)) + return ob.TransportResponse( + 200, {"result": {"run_test_id": "run-test-1", "scenarios": []}}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "passing", scenario_key="passing", scenario_id="", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + transport = MismatchedProvisionTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "platform_sync" + assert failure["code"] == "scenario_preallocation_failed" + + assert transport.receipts == {} # the scheduler never ran -- registration failed first + # `begin` must never have been attempted -- the provision-side guard stops it first. + assert not any(body.get("operation") == "begin" for _, body in transport.scenarios_calls) + + asyncio.run(scenario()) + + def test_injected_scenario_source_always_wins_over_the_bundle_adapter() -> None: # item 4: even when the bundle ALSO carries a valid `scenarios/` directory, an explicitly # injected `ScenarioSource` must be used untouched -- the presence test only ever applies to @@ -2534,6 +2687,7 @@ async def scenario() -> None: test_cancel_state_reads_reason_from_file, test_world_pool_serializes_concurrent_provider_calls_end_to_end, test_scenarios_client_provision_unwraps_the_result_envelope, + test_scenarios_client_provision_and_begin_hit_the_same_single_url, test_scenarios_client_fencing_latches_the_shared_channel_state, test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event, test_preflight_rejection_reaches_a_failed_terminal_event_before_any_provision, @@ -2578,7 +2732,9 @@ async def scenario() -> None: test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present, test_empty_scenario_key_from_bundle_document_fails_cleanly_via_existing_validation, test_mutation_adapter_off_makes_the_e2e_test_fail, - test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema, + test_bundle_scenario_id_is_assigned_by_registration_and_receipt_now_delivers, + test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely, + test_registration_response_mismatch_reaches_the_typed_platform_sync_terminal, test_injected_scenario_source_always_wins_over_the_bundle_adapter, test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure, test_module_level_sys_exit_three_in_setup_does_not_hijack_the_guests_exit_code, diff --git a/tests/harness/test_scenario_source.py b/tests/harness/test_scenario_source.py index 718baf4c..22b4dc8e 100644 --- a/tests/harness/test_scenario_source.py +++ b/tests/harness/test_scenario_source.py @@ -31,6 +31,7 @@ import pytest from fi.alk.harness import scenario_source as ss +from fi.alk.harness.hosted_entrypoint import ScenarioPreallocationError from fi.alk.harness.hosted_scheduler import Call, CallOutcome, HostedScheduler, WorldPool from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState @@ -40,6 +41,15 @@ # ================================================================================================= +@dataclass +class _FakeJob: + """Stands in for `HarnessJob` wherever only `.run_id` is read (p13: + `BundleScenarioSource.build` uses it as the platform-facing provision run name) -- avoids + importing the real pydantic model into this module purely for one attribute.""" + + run_id: str = "job-1" + + def _write_scenario( scenarios_root: Path, name: str, @@ -940,9 +950,19 @@ async def scenario() -> None: root = tmp_path / ss.SCENARIOS_DIRNAME _write_scenario(root, "s1", scenario_key="s1", sub_goals=[]) source = ss.BundleScenarioSource() - scenarios = await source.build( - object(), object(), object(), pool=object(), world_factory=object(), bundle_dir=tmp_path - ) + + # p13: `build()` now calls `register_with_platform` after load -- this test is about the + # R1-5 timeout BUDGET specifically, not registration, so registration is stubbed to a + # passthrough (registration's own behavior is covered separately, below). + async def _passthrough(scenarios_client, scenarios, *, run_name): + del scenarios_client, run_name + return scenarios + + with mock.patch.object(ss, "register_with_platform", _passthrough): + scenarios = await source.build( + _FakeJob(run_id="job-1"), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) assert [s.scenario_key for s in scenarios] == ["s1"] asyncio.run(scenario()) @@ -1040,3 +1060,356 @@ def _synthesizes_key(folder: Path): restored = ss.load_scenarios(tmp_path)[0] assert restored.scenario_key == "" # confirms the patch was fully undone + + +# ================================================================================================= +# p13 -- scenario pre-allocation (`register_with_platform`), wired against the platform's actual +# route (a single `POST .../scenarios/`, discriminated by a body-level `operation` field, keyed +# provision response, full-set `begin`) rather than Karthik's documented two-path/position-ordered +# shape -- see `ScenariosClient`'s and `register_with_platform`'s own docstrings for the file:line +# evidence, and reports/p13-worker-r2.md CONTRACT NOTES for where the two disagree. +# ================================================================================================= + + +def _scenario(key: str, *, scenario_id: str = "") -> ss._CompiledScenario: + return ss._CompiledScenario( + scenario_key=key, scenario_id=scenario_id, sub_goals=(), setup=lambda w: None, + ready=lambda w: None, + ) + + +@dataclass +class _FakeScenariosClient: + """Stands in for `hosted_entrypoint.ScenariosClient` -- records every payload it is called + with (so a test can assert on exactly what `register_with_platform` sends) and returns a + canned `{"result": {...}}`-unwrapped body per call, matching what the real + `ScenariosClient._post` already hands back (the envelope itself is that class's concern, not + this module's -- see `test_scenarios_client_provision_unwraps_the_result_envelope` in + `test_hosted_entrypoint.py`).""" + + provision_response: dict[str, Any] + begin_response: dict[str, Any] = field( + default_factory=lambda: {"test_execution_id": "exec-1", "scenarios": []} + ) + provision_error: Exception | None = None + begin_error: Exception | None = None + provision_calls: list[dict[str, Any]] = field(default_factory=list) + begin_calls: list[dict[str, Any]] = field(default_factory=list) + + def provision(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + del deadline + self.provision_calls.append(payload) + if self.provision_error is not None: + raise self.provision_error + return self.provision_response + + def begin(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + del deadline + self.begin_calls.append(payload) + if self.begin_error is not None: + raise self.begin_error + return self.begin_response + + +# ------------------------------------------------------------------------------------------------- +# `BundleScenarioSource.build` wiring -- registration runs after load, before the scheduler; a +# LOCALLY-detectable defect (empty scenario_key) must never reach the network first. +# ------------------------------------------------------------------------------------------------- + + +def test_build_rejects_empty_scenario_key_before_calling_register_with_platform( + tmp_path: Path, +) -> None: + # An empty scenario_key is carried verbatim off the document (this module's own LAYOUT + # DECISION -- never synthesized), but `hosted_entrypoint.py`'s own `_validate_scenarios` would + # reject it downstream as a deterministic, `environment`-domain content defect. Checked here, + # before `register_with_platform` is ever called, so that existing classification wins over a + # round trip that would only rediscover the same defect as a `platform_sync` failure instead. + async def scenario() -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", sub_goals=[]) + source = ss.BundleScenarioSource() + + register_calls: list[Any] = [] + + async def _spy(scenarios_client, scenarios, *, run_name): + register_calls.append((scenarios_client, scenarios, run_name)) + return scenarios + + with mock.patch.object(ss, "register_with_platform", _spy): + with pytest.raises(ss.ScenarioDocumentInvalid, match="scenario_key"): + await source.build( + _FakeJob(), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) + assert register_calls == [] # never reached the network + + asyncio.run(scenario()) + + +# ------------------------------------------------------------------------------------------------- +# Payload shape -- quote-driven against Azain's serializers (file:line in each docstring/comment). +# ------------------------------------------------------------------------------------------------- + + +def test_provision_payload_only_sends_fields_azains_serializer_declares() -> None: + # HarnessProvisionPersonaSerializer (futureagi/simulate/serializers/hosted_harness.py:168-174): + # scenario_key is the only REQUIRED field; name/role/situation/outcome/persona are all + # `required=False`. `_CompiledScenario` carries none of the optional ones (this module's own + # LAYOUT DECISION -- see the module docstring), so the payload must send bare scenario_key + # only, never a synthesized value for a field this reader does not have. + scenarios = (_scenario("book_a_ride"), _scenario("cancel_ride")) + payload = ss._provision_payload("run-1", scenarios) + assert payload == { + "operation": "provision", + "name": "run-1", + "personas": [{"scenario_key": "book_a_ride"}, {"scenario_key": "cancel_ride"}], + } + for persona in payload["personas"]: + assert set(persona) == {"scenario_key"} + + +def test_begin_payload_carries_operation_run_test_id_and_the_full_key_set() -> None: + # HarnessScenarioBeginSerializer (futureagi/simulate/serializers/hosted_harness.py:193-198): + # `scenario_keys` is REQUIRED and `allow_empty=False` -- the full sealed set every time, never + # a subset (`begin_scenarios`, services/hosted_harness.py:323-329, 409s on anything less). + scenarios = (_scenario("a"), _scenario("b")) + payload = ss._begin_payload("run-test-1", scenarios) + assert payload == { + "operation": "begin", "run_test_id": "run-test-1", "scenario_keys": ["a", "b"], + } + + +# ------------------------------------------------------------------------------------------------- +# Keyed-response parsing (`_scenario_ids_by_key`) -- dict lookup by scenario_key, never a +# positional zip; every malformed/mismatched shape raises rather than returning a partial mapping. +# ------------------------------------------------------------------------------------------------- + + +def test_scenario_ids_by_key_matches_regardless_of_response_order() -> None: + # The platform's response order is not guaranteed to match submission order -- this is the + # load-bearing case a positional zip would get wrong (mutation-killed below). + submitted = (_scenario("a"), _scenario("b")) + reordered_response = [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ] + assert ss._scenario_ids_by_key(submitted, reordered_response) == { + "a": "platform-a", "b": "platform-b", + } + + +def test_scenario_ids_by_key_raises_typed_error_for_non_list_response() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), {"not": "a list"}) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +def test_scenario_ids_by_key_raises_typed_error_for_a_non_object_entry() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), ["not an object"]) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +def test_scenario_ids_by_key_raises_typed_error_for_unknown_key() -> None: + submitted = (_scenario("a"),) + raw = [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "never-submitted", "scenario_id": "platform-x"}, + ] + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_unknown_key" + + +def test_scenario_ids_by_key_raises_typed_error_for_missing_key() -> None: + submitted = (_scenario("a"), _scenario("b")) + raw = [{"scenario_key": "a", "scenario_id": "platform-a"}] # "b" never comes back + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_missing" + + +def test_scenario_ids_by_key_raises_typed_error_for_duplicate_key() -> None: + submitted = (_scenario("a"),) + raw = [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "a", "scenario_id": "platform-a-again"}, + ] + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_duplicate_key" + + +def test_scenario_ids_by_key_raises_typed_error_for_empty_scenario_id() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), [{"scenario_key": "a", "scenario_id": ""}]) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +# ------------------------------------------------------------------------------------------------- +# `register_with_platform` -- full sequence: provision -> match (guards) -> begin -> assign. +# ------------------------------------------------------------------------------------------------- + + +def test_register_with_platform_assigns_platform_ids_and_begins_the_full_set() -> None: + async def scenario() -> None: + submitted = (_scenario("a"), _scenario("b")) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + # Deliberately out of order relative to `submitted` -- proves the match is by key. + "scenarios": [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ], + }, + ) + result = await ss.register_with_platform(client, submitted, run_name="run-1") + + assert [s.scenario_key for s in result] == ["a", "b"] # submission order preserved + assert [s.scenario_id for s in result] == ["platform-a", "platform-b"] # matched by key + assert result[0].setup is submitted[0].setup # untouched fields carried through verbatim + assert result[0].ready is submitted[0].ready + assert result[0].sub_goals is submitted[0].sub_goals + + assert client.provision_calls == [ + { + "operation": "provision", "name": "run-1", + "personas": [{"scenario_key": "a"}, {"scenario_key": "b"}], + } + ] + assert client.begin_calls == [ + {"operation": "begin", "run_test_id": "run-test-1", "scenario_keys": ["a", "b"]}, + ] + + asyncio.run(scenario()) + + +def test_register_with_platform_guard_failure_never_calls_begin() -> None: + # "never partial assignment": a guard failure during provision-response parsing must stop + # BEFORE begin is ever called -- no scenario in this batch gets sealed for execution against a + # registration the client-side guard has already rejected. + async def scenario() -> None: + submitted = (_scenario("a"), _scenario("b")) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], # "b" missing + }, + ) + with pytest.raises(ScenarioPreallocationError) as exc_info: + await ss.register_with_platform(client, submitted, run_name="run-1") + assert exc_info.value.error.code == "scenario_registration_missing" + assert client.begin_calls == [] + + asyncio.run(scenario()) + + +def test_register_with_platform_missing_run_test_id_is_a_typed_failure_before_begin() -> None: + async def scenario() -> None: + submitted = (_scenario("a"),) + client = _FakeScenariosClient( + provision_response={ + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], + }, + ) + with pytest.raises(ScenarioPreallocationError) as exc_info: + await ss.register_with_platform(client, submitted, run_name="run-1") + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + assert client.begin_calls == [] + + asyncio.run(scenario()) + + +# ------------------------------------------------------------------------------------------------- +# Mutations (p13 work item 5). +# ------------------------------------------------------------------------------------------------- + + +def test_mutation_positional_zip_matching_is_killed() -> None: + # Mutant: `_scenario_ids_by_key` replaced with a positional zip (Karthik's documented shape -- + # not what the platform actually returns). A reordered response must silently mismatch ids + # under the mutant; the real (key-matching) implementation must not. + submitted = (_scenario("a"), _scenario("b")) + reordered_response = [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ] + + real = ss._scenario_ids_by_key(submitted, reordered_response) + assert real == {"a": "platform-a", "b": "platform-b"} # baseline: correct regardless of order + + def _positional_zip_mutant(submitted, raw_scenarios): + return { + scenario.scenario_key: entry["scenario_id"] + for scenario, entry in zip(submitted, raw_scenarios, strict=True) + } + + with mock.patch.object(ss, "_scenario_ids_by_key", _positional_zip_mutant): + mutant = ss._scenario_ids_by_key(submitted, reordered_response) + assert mutant == {"a": "platform-b", "b": "platform-a"} # mutant's defect: swapped ids + assert mutant != real + + restored = ss._scenario_ids_by_key(submitted, reordered_response) + assert restored == real # confirms the patch was fully undone + + +def test_mutation_missing_scenario_guard_removed_is_killed() -> None: + # Mutant: the "every submitted key must appear in the response" check deleted from + # `_scenario_ids_by_key` -- as if a job with fewer provisioned scenarios than requested were + # silently accepted instead of failing the whole registration. + submitted = (_scenario("a"), _scenario("b")) + incomplete_response = [{"scenario_key": "a", "scenario_id": "platform-a"}] # "b" never comes back + + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, incomplete_response) # baseline: the real guard catches it + assert exc_info.value.error.code == "scenario_registration_missing" + + def _no_missing_guard_mutant(submitted, raw_scenarios): + del submitted + return {entry["scenario_key"]: entry["scenario_id"] for entry in raw_scenarios} + + with mock.patch.object(ss, "_scenario_ids_by_key", _no_missing_guard_mutant): + mutant = ss._scenario_ids_by_key(submitted, incomplete_response) # mutant: no longer raises + assert "b" not in mutant # confirms the mutant's defect: an incomplete mapping got through + + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, incomplete_response) # restored + assert exc_info.value.error.code == "scenario_registration_missing" + + +def test_mutation_id_assignment_skipped_is_killed() -> None: + # Mutant: `register_with_platform`'s final `replace(scenario, scenario_id=...)` step deleted -- + # provision/begin both still run (a response-shape bug would not be caught by this mutant), but + # the scenarios handed back to the scheduler never actually carry the platform's id. + async def scenario() -> None: + submitted = (_scenario("a"),) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], + }, + ) + + real = await ss.register_with_platform(client, submitted, run_name="run-1") + assert real[0].scenario_id == "platform-a" # baseline: the real fix assigns it + + async def _skip_assignment_mutant(scenarios_client, scenarios, *, run_name): + provision_result = await asyncio.to_thread( + scenarios_client.provision, ss._provision_payload(run_name, scenarios) + ) + await asyncio.to_thread( + scenarios_client.begin, + ss._begin_payload(provision_result["run_test_id"], scenarios), + ) + return scenarios # mutant's defect: returned VERBATIM, ids never merged in + + with mock.patch.object(ss, "register_with_platform", _skip_assignment_mutant): + mutant = await ss.register_with_platform(client, submitted, run_name="run-1") + assert mutant[0].scenario_id == "" # mutant's defect: id never assigned + + restored = await ss.register_with_platform(client, submitted, run_name="run-1") + assert restored[0].scenario_id == "platform-a" # confirms the patch was fully undone + + asyncio.run(scenario()) From ef4c5a72e8dafa4296b6ba010cc2950c332fe3a0 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 26 Aug 2026 18:53:56 +0530 Subject: [PATCH 17/20] =?UTF-8?q?feat(harness):=20real-voice=20CallRunner?= =?UTF-8?q?=20=E2=80=94=20the=20hosted=20guest=20places=20the=20call?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the typed not-wired call seam with a real LiveKit runner when the job's connector is livekit: pre-dial validation of credentials and dispatch identity, an in-process SimulationSpec drive of the voice engine, deterministic per-scenario room naming, and a runner-owned call budget (config voice_call_timeout_seconds, default 300). Evidence follows the bundle-declared seam: tool_trace reads the world's database endpoint after the call; http_tool has no guest-side capture surface today and stays a typed stop rather than an invented proxy. Transcripts and recordings upload through the artifacts channel before the receipt references them, and artifact-level refusals degrade to null. Failure semantics are three distinct paths, each pinned by tests: an agent that never joins retires the world; any post-dial failure keeps its measured timing on the receipt; a silent zero-turn agent surfaces as missing evidence rather than a graded verdict or a false infrastructure failure. Voice credentials are captured from the job's secrets before the provisioner deletes them and exported once for the engine's own reads; values never appear in logs and stay inside the redaction set. Cold reviews: round 1 found 1 High (test-honesty) + 2 Medium, all fixed; round 2 CLEARS — 18 mutation runs killed or validated, 863 passed twice. Signed-off-by: khushalsonawat --- src/fi/alk/harness/call_runner.py | 729 +++++++++++++++++++++ src/fi/alk/harness/hosted_entrypoint.py | 82 ++- tests/harness/test_call_runner.py | 799 ++++++++++++++++++++++++ tests/harness/test_hosted_entrypoint.py | 151 ++++- 4 files changed, 1749 insertions(+), 12 deletions(-) create mode 100644 src/fi/alk/harness/call_runner.py create mode 100644 tests/harness/test_call_runner.py diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py new file mode 100644 index 00000000..2b249da6 --- /dev/null +++ b/src/fi/alk/harness/call_runner.py @@ -0,0 +1,729 @@ +"""The hosted lane's `CallRunner` — places one simulated LiveKit voice call and reports what +happened, satisfying `hosted_scheduler.CallRunner` exactly. + +Three sub-systems (world-handle-interface.md, hosted-execution-seams.md v1.15 §2a): + +1. **Placing the call.** The customer agent is already running INSIDE the Daytona sandbox, as a + world process the bundle's provisioner spawned (`process_runtime.py`) and registered with + LiveKit cloud under `LIVEKIT_AGENT_NAME=agent-w{WORLD_INDEX}`-style identity. This runner never + starts or manages that process — it drives `SimulationRunner` IN-PROCESS with a directly-built + `SimulationSpec` that dials the already-registered identity, mirroring `run/sdk_voice.py:: + build_spec` field-for-field but sourcing values from job config and the bundle's own scenario + document instead of `HARNESS_*` env vars (the local-only webhook/subprocess plumbing + `run/call.py`/`run/live.py` use is neither available nor appropriate in the guest). +2. **Collecting evidence.** The bundle declares exactly one `runtime.evidence_seam`: + `http_tool` or `tool_trace`. `http_tool` has NO guest-side capture surface anywhere in this + repo today (see `_collect_http_tool_calls`'s docstring — a verified finding, not an assumption) + and is intentionally left returning zero calls rather than inventing a capture proxy. + `tool_trace` is read from the world's own postgres database against an unpinned, isolated + convention (see `_collect_tool_trace_calls`'s docstring). Either way, zero calls captured is + never fabricated into something else — the scheduler's own `evidence_missing` retry-once policy + is the contract-correct handling for "no evidence." +3. **Uploading artifacts.** The transcript and any produced recordings are uploaded through the + adapter's `upload_artifact` (content-addressed, budget/level-gated, returns `None` on refusal — + never an exception) BEFORE this runner returns, so `CallOutcome.transcript_artifact`/ + `recording_artifacts` only ever carry ids the platform has already acked. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, Mapping, Protocol + +from fi import simulate +from fi.simulate.runtime import ( + AgentEndpointSpec, + EnvironmentSpec, + ExecutionPolicy, + SimulationSpec, + SimulatorPolicySpec, + TimeoutPolicy, + new_run_id, +) +from fi.simulate.runtime.report import SimulationReport +from fi.simulate.runtime.run import TestCaseStatus +from fi.simulate.runtime.runner import SimulationRunner + +from .bundle_v2 import EvidenceSeam +from .hosted_scheduler import CallAborted, CallOutcome +from .hosted_scheduler import Scenario as HostedScenario +from .job import HarnessJob +from .outbound import ArtifactKind, format_rfc3339_millis +from .process_runtime import EnvironmentRuntime +from .world.errors import WorldUnavailable +from .world.runtime import Call + +logger = logging.getLogger(__name__) + +# --- credential aliases / config keys a voice job must carry --- + +LIVEKIT_API_KEY_ALIAS = "LIVEKIT_API_KEY" +LIVEKIT_API_SECRET_ALIAS = "LIVEKIT_API_SECRET" +DEEPGRAM_API_KEY_ALIAS = "DEEPGRAM_API_KEY" +GEMINI_API_KEY_ALIAS = "GEMINI_API_KEY" +GOOGLE_API_KEY_ALIAS = "GOOGLE_API_KEY" # either this or GEMINI_API_KEY_ALIAS satisfies the LLM leg + +LIVEKIT_URL_CONFIG_KEY = "livekit_url" +CALL_TIMEOUT_CONFIG_KEY = "voice_call_timeout_seconds" + +_DEFAULT_CALL_TIMEOUT_SECONDS = 300.0 + +# sdk_voice.py::build_spec's own phase-overhead constants, reused verbatim so this runner's +# outer budget composes with the SDK's internal one the same way the local template does. +_CONNECT_TIMEOUT_SECONDS = 60.0 +_READINESS_TIMEOUT_SECONDS = 120.0 +_CLEANUP_TIMEOUT_SECONDS = 30.0 +_RUN_SECONDS_PAD_SECONDS = 60.0 +# Headroom beyond `spec.execution.timeout.run_seconds` -- SimulationRunner.run() already wraps +# `plugin.run(...)` in its OWN `asyncio.wait_for(..., timeout=spec.execution.timeout.run_seconds)` +# (runner.py) and catches that TimeoutError into a graceful `SimulationReport(status=TIMED_OUT)`. +# This runner's own outer wait_for must stay LARGER than that so the SDK's internal timeout fires +# first in the ordinary case; it only ever fires itself for a genuinely hung SDK (a real post-dial +# machinery failure) -- a runner-owned asyncio.wait_for as the last-resort bound. +_OUTER_WAIT_FOR_PAD_SECONDS = 60.0 + +# Unpinned by any contract and no producer exists yet. Isolated as one +# constant + two functions (`_clear_tool_trace_calls`, `_collect_tool_trace_calls`) so a real +# producer's disagreement on the name/shape is a one-line change. +_TOOL_TRACE_TABLE = "_alk_tool_trace" + +_RESULT_TRUNCATE_CHARS = 2000 + +# The real engine's zero-turn "agent joined but never spoke" failure codes (engines/livekit.py:: +# _conversation_outcome) -- see `_translate_report`'s `is_silent_agent` gate for why these two, and +# only at zero turns, get mapped to a normal CallOutcome instead of a CallAborted. +_SILENT_AGENT_FAILURE_CODES = frozenset({"no_conversation", "conversation_silence_timeout"}) + + +# --- collaborator seams (named, injectable test boundaries) ----------------------------------- + + +class ArtifactUploader(Protocol): + """Narrow slice of `hosted_entrypoint.OutboundAdapter` -- avoids importing that module here + (it imports THIS module's factory to wire the real CallRunner; importing it back would be + circular).""" + + async def upload_artifact( + self, + data: bytes, + *, + kind: ArtifactKind, + scenario_key: str | None = None, + deadline: float | None = None, + ) -> str | None: ... + + +PlaceCall = Callable[[SimulationSpec], Awaitable[SimulationReport]] + + +async def _default_place_call(spec: SimulationSpec) -> SimulationReport: + return await SimulationRunner().run(spec) + + +@dataclass(frozen=True) +class CallRunnerContext: + """Everything `hosted_entrypoint.py`'s `run_job` already has in scope by the wiring point + (~1662) that the real `CallRunnerImpl` needs but the bare `CallRunner` protocol signature + (`run(scenario, runtime)`) has no room to carry. Threaded through the EXTENDED + `build_call_runner(adapter, context)` seam.""" + + job: HarnessJob + bundle_dir: Path + work_directory: Path + evidence_seam: EvidenceSeam | None + target_provider_secret_values: Mapping[str, str] + attempt_number: int + + +# --- pre-dial validation ----------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _MissingVoiceConfig: + aliases: tuple[str, ...] + config_keys: tuple[str, ...] + + def message(self) -> str: + parts = [] + if self.aliases: + parts.append("secrets=" + ",".join(self.aliases)) + if self.config_keys: + parts.append("config=" + ",".join(self.config_keys)) + return "voice_capability_unavailable: missing " + "; ".join(parts) + + +def _check_config( + job: HarnessJob, target_provider_secret_values: Mapping[str, str] +) -> _MissingVoiceConfig | None: + missing_aliases = [ + alias + for alias in (LIVEKIT_API_KEY_ALIAS, LIVEKIT_API_SECRET_ALIAS, DEEPGRAM_API_KEY_ALIAS) + if not target_provider_secret_values.get(alias) + ] + if not target_provider_secret_values.get( + GEMINI_API_KEY_ALIAS + ) and not target_provider_secret_values.get(GOOGLE_API_KEY_ALIAS): + missing_aliases.append(f"{GEMINI_API_KEY_ALIAS}_or_{GOOGLE_API_KEY_ALIAS}") + missing_config_keys = ( + [] if job.agent.config.get(LIVEKIT_URL_CONFIG_KEY) else [LIVEKIT_URL_CONFIG_KEY] + ) + if not missing_aliases and not missing_config_keys: + return None + return _MissingVoiceConfig(tuple(missing_aliases), tuple(missing_config_keys)) + + +def _dispatch_agent_name(runtime: EnvironmentRuntime) -> str | None: + """The ONLY place this repo reads the dispatch-identity metadata key, so a + change to the key name/convention is a one-line adapt. `EnvironmentRuntime.metadata` defaults to `{}` and nothing + in `process_runtime.py` populates it yet -- every real "livekit" job + hits the caller's typed `CallAborted` below until a producer lands.""" + value = runtime.metadata.get("livekit_agent_name") + return value.strip() if isinstance(value, str) and value.strip() else None + + +# --- scenario document re-read (the _CompiledScenario the scheduler hands over carries no +# persona/instruction -- scenario_source.py:170-184's deliberately narrow Scenario-protocol +# shape) ------------------------------------------------------------------------------------ + + +class _ScenarioDocumentUnavailable(RuntimeError): + pass + + +def _read_scenario_document(bundle_dir: Path, scenario_key: str) -> dict[str, Any]: + """Re-reads `scenarios//scenario.json` from the bundle, matched by the document's OWN + `scenario_key` field -- never the folder name (`scenario_source.py`'s own convention; the two + are not guaranteed to match).""" + root = bundle_dir / "scenarios" + if not root.is_dir(): + raise _ScenarioDocumentUnavailable(f"no {root} directory in this bundle") + try: + children = sorted(root.iterdir()) + except OSError as exc: + raise _ScenarioDocumentUnavailable(f"cannot list {root}: {exc}") from exc + for child in children: + if not child.is_dir(): + continue + doc_path = child / "scenario.json" + if not doc_path.is_file(): + continue + try: + body = json.loads(doc_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(body, dict) and body.get("scenario_key") == scenario_key: + instruction = body.get("instruction") + if not isinstance(instruction, str) or not instruction.strip(): + raise _ScenarioDocumentUnavailable( + f"{child.name}/scenario.json has no non-empty instruction" + ) + return body + raise _ScenarioDocumentUnavailable( + f"no scenario.json under {root} carries scenario_key={scenario_key!r}" + ) + + +# --- deterministic room naming (asserted verbatim by tests/harness/test_call_runner.py). WHY this +# is a PREFIX guarantee, not a full-match one: in managed room_mode, engines/livekit.py:: +# _resolve_room_name appends its own `-{invocation_id}-{test_case_id[-12:]}` suffix unless +# `room_name_verbatim` is set (which this runner does not set) -- the scheme below still gives +# every call a unique, deterministic, greppable prefix; only the exact wire-level name is not this +# string verbatim. ------------------------------------------------------------------------------- + + +def _room_name( + *, job_id: str, attempt_number: int, scenario_key: str, scenario_attempt: int +) -> str: + return f"harness-{job_id[:8]}-a{attempt_number}-{scenario_key}-s{scenario_attempt}" + + +def _duration_ms(started_at: datetime, ended_at: datetime) -> int: + return max(0, int((ended_at - started_at).total_seconds() * 1000)) + + +# --- SimulationSpec construction (mirrors run/sdk_voice.py::build_spec field-for-field; values +# come from job config / the re-read scenario document instead of HARNESS_* env vars) --------- + + +def _simulator_definition() -> Any: + """Mirrors `sdk_voice.py::_simulator()`'s own shipped defaults + exactly (google/gemini LLM, deepgram STT+TTS) -- the only provider combination this runner's + pre-dial credential check (`_check_config`) validates.""" + return simulate.SimulatorAgentDefinition( + llm={"provider": "google", "model": "gemini-2.5-flash-lite", "temperature": 0.35}, + stt={"provider": "deepgram", "model": "nova-2", "language": "en"}, + tts={"provider": "deepgram", "model": "aura-asteria-en", "voice": "aura-asteria-en"}, + instructions=( + "Act as the customer described by the scenario. Speak naturally and briefly. " + "Use only the supplied facts and never invent account, address, payment, or " + "verification data. Do not volunteer private data: agree when asked whether a " + "verification code should be sent, and disclose the actual code only after the " + "agent says it was sent and explicitly asks you to read it. Answer repair questions " + "with the missing fact, not by restarting the request. Never repeat the same answer " + "more than twice. When the requested outcome is complete, thank the agent and end " + "the call." + ), + allow_interruptions=True, + ) + + +def _scenario_spec(doc: Mapping[str, Any]) -> Any: + """Mirrors `sdk_voice.py::_scenario()`'s own transformation exactly, sourced from the re-read + scenario document instead of `HARNESS_*` env vars.""" + fixture = doc.get("fixture") if isinstance(doc.get("fixture"), dict) else {} + persona = dict(doc.get("persona") or {}) + persona["role"] = "customer" + metadata = dict(persona.get("metadata") or {}) + if fixture.get("phone"): + metadata["caller_phone"] = str(fixture["phone"]) + persona["metadata"] = metadata + knowledge = [ + { + "key": str(key), + "value": json.dumps(value, ensure_ascii=False, default=str), + "disclosure": "on_request", + } + for key, value in fixture.items() + if key != "origin" + ] + persona_model = simulate.Persona( + persona=persona, + situation=doc["instruction"], + outcome=(doc.get("tests") or "Complete the requested task and close naturally."), + knowledge=knowledge, + behavior_policy={ + "disclosure_policy": 0.72, + "cooperation_bounds": 0.9, + "repair_propensity": 0.85, + }, + ) + return simulate.Scenario( + name=str(doc.get("scenario_key") or doc.get("name") or "harness-voice"), + dataset=[persona_model], + ) + + +def _build_spec( + *, + run_id: str, + room_name: str, + agent_name: str, + doc: Mapping[str, Any], + livekit_url: str, + call_timeout_seconds: float, + run_seconds: float, + recordings_root: Path, +) -> SimulationSpec: + recording_dir = recordings_root / run_id / "recordings" + params = { + "record_audio": True, + "recording_root": str(recording_dir), + "recording_case_directory": str(recording_dir), + "min_turn_messages": 6, + "max_seconds": call_timeout_seconds, + "connect_timeout": _CONNECT_TIMEOUT_SECONDS, + "readiness_timeout": _READINESS_TIMEOUT_SECONDS, + "cleanup_timeout": _CLEANUP_TIMEOUT_SECONDS, + "conversation_direction": "agent_first", + "agent_first_silence_timeout_seconds": 45.0, + } + agent = simulate.AgentDefinition( + name="harness-livekit-target", + agent_name=agent_name, + system_prompt=doc["instruction"], + transport={"kind": "webrtc"}, + ) + runtime_spec = simulate.LiveKitSimulatorRuntime( + url=livekit_url, room_name=room_name, room_mode="managed", + ) + return SimulationSpec( + run_id=run_id, + environment=EnvironmentSpec( + adapter="voice", + world_kind="voice_telephony", + config={ + "agent_definition": agent.model_dump(mode="json", exclude_none=True), + "livekit_runtime": runtime_spec.model_dump(mode="json", exclude_none=True), + "simulator": _simulator_definition().model_dump(mode="json", exclude_none=True), + "params": params, + }, + ), + target=AgentEndpointSpec(adapter="webrtc"), + simulator=SimulatorPolicySpec(adapter="livekit_simulator"), + scenario=_scenario_spec(doc), + execution=ExecutionPolicy( + direction="agent_first", timeout=TimeoutPolicy(run_seconds=run_seconds) + ), + ) + + +# --- evidence collection ----------------------------------------------------------------------- + + +def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> Any | None: + """Protocol-based lookup, matching `hosted_entrypoint.py::_find_postgres_endpoint`'s own + already-correct convention -- capability slugs are bundle-author-chosen (`build_endpoints`, + process_runtime.py:318-339), never a fixed key, so a hardcoded `endpoints["database"]` would + break for any bundle that names its capability slug differently. Re-implemented locally rather + than imported: importing from `hosted_entrypoint.py` here would be circular (it imports this + module's factory).""" + for endpoint in runtime.endpoints.values(): + if endpoint.protocol == "postgres": + return endpoint + return None + + +def _collect_http_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: + """No guest-side capture surface exists anywhere in this repo for the + `http_tool` evidence seam. Verified, not assumed: `world/handle.py::HostedWorld.call()` + raises `WorldUnavailable` unconditionally with a docstring stating the wire format "is not + pinned anywhere in the contracts yet"; `process_runtime.py`'s own `provision()` signature + comment says "evidence-seam wiring is out of this phase's scope"; no `TOOLS_API_URL` wiring + exists in the hosted lane at all (the local lane's `ProvisionedWorld`/`TOOLS_API_URL` mechanism + lives in `provision.py`/`world/provisioned.py`, out of scope here and inapplicable to the guest + regardless). Deliberately stopped rather than inventing a capture proxy: a job whose bundle + declares `evidence_seam: http_tool` reads zero calls every time, which the scheduler's own + `evidence_missing` retry-once policy turns into the correct, honest outcome -- never a crash, + never fabricated evidence.""" + del runtime + return () + + +def _clear_tool_trace_calls(dsn: str) -> None: + """world-handle-interface.md: "setup's tool calls are NOT evidence (the runner clears them + before the call starts, as the local runner does)" -- the local runner's analog is + `world.calls = []` right before dialing (`run/simulation.py`). Best-effort: a missing table (no + producer yet) or any connection error is swallowed, never raised. Clearing + is housekeeping, not a correctness requirement, while nothing writes this table yet; once a + real producer lands this stops being a no-op automatically.""" + try: + import psycopg + + with psycopg.connect(dsn, autocommit=True, connect_timeout=5) as connection: + connection.execute(f'DELETE FROM "{_TOOL_TRACE_TABLE}"') # noqa: S608 - fixed identifier, no interpolated user input + except Exception as exc: # noqa: BLE001 - best-effort housekeeping only, never a call-blocking failure + # WHY: never log exc_info / str(exc) here -- a psycopg connection failure embeds the raw + # DSN (including the world DB password) in its own exception message; only the exception + # TYPE is safe for a local log line. + logger.debug("tool_trace clear skipped (table likely absent): %s", type(exc).__name__) + + +def _collect_tool_trace_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: + """`_alk_tool_trace`'s name and column shape are an isolated local + convention -- unpinned by any contract (the only harness-reserved table anywhere in this + repo is `_alk_conformance`, unrelated), no producer exists yet. Isolated in this one function + (+ `_clear_tool_trace_calls`) so a real producer's disagreement on the name/shape is a one-line + change. Any failure (missing table, connection refused, malformed row) degrades to `()` -- + never a crash, never fabricated evidence, matching `_collect_http_tool_calls`'s stopped + behavior above.""" + endpoint = _find_postgres_endpoint(runtime) + if endpoint is None: + return () + try: + import psycopg + + with psycopg.connect( + endpoint.address, + autocommit=True, + connect_timeout=5, + options="-c default_transaction_read_only=on", + ) as connection: + cursor = connection.execute( + f'SELECT name, arguments, result, ok, error, at FROM "{_TOOL_TRACE_TABLE}" ' # noqa: S608 + "ORDER BY at ASC" + ) + rows = cursor.fetchall() + columns = [description[0] for description in cursor.description or []] + except Exception as exc: # noqa: BLE001 - missing table / connection failure -> no evidence, not a crash + # WHY: same DSN-in-exception-message risk as `_clear_tool_trace_calls` above -- log only + # the exception TYPE, never exc_info/str(exc), which can carry the world DB password. + logger.debug("tool_trace read failed; treating as no evidence: %s", type(exc).__name__) + return () + + calls: list[Call] = [] + for row in rows: + record = dict(zip(columns, row, strict=True)) + name = record.get("name") + if not isinstance(name, str) or not name: + continue + arguments = record.get("arguments") + if not isinstance(arguments, dict): + arguments = {} + ok = bool(record.get("ok", True)) + raw_result = record.get("result") + if isinstance(raw_result, str): + result: Any = _truncate(raw_result) + else: + # Already parsed JSON (dict/list/etc, psycopg's own jsonb decoding) -- per + # world-handle-interface.md, only the STRING form is truncated at 2000 chars. + result = raw_result + error = _truncate(str(record.get("error") or "")) + raw_at = record.get("at") + at = float(raw_at) if isinstance(raw_at, (int, float)) else 0.0 + calls.append( + Call(name=name, arguments=arguments, result=result, ok=ok, error=error, refused=not ok, at=at) + ) + return tuple(calls) + + +def _truncate(value: str, *, limit: int = _RESULT_TRUNCATE_CHARS) -> str: + return value if len(value) <= limit else value[:limit] + + +# --- the runner ---------------------------------------------------------------------------- + + +class CallRunnerImpl: + """Satisfies `hosted_scheduler.CallRunner`. See the module docstring for the three + sub-systems this class implements.""" + + def __init__( + self, + adapter: ArtifactUploader, + context: CallRunnerContext, + *, + place_call: PlaceCall | None = None, + environ: dict[str, str] | None = None, + ) -> None: + self._adapter = adapter + self._context = context + self._place_call = place_call or _default_place_call + # WHY: the underlying LiveKit engine reads these directly via `os.environ.get(...)` deep + # inside `engines/livekit.py` / `livekit_models.py` -- they are NOT `SimulationSpec` + # fields, so there is no other way to hand them over. Exported ONCE here, at construction, + # not per-call: the values are job-level (the same secret for every scenario/attempt on + # this job) and W>1 means each world's CallRunner.run() executes inside this SAME guest + # process but against a per-world SANDBOXED agent process reached over the network -- no + # other in-process worker ever races this write, so one job-level export is race-free. + target_environ = os.environ if environ is None else environ + for alias in ( + LIVEKIT_API_KEY_ALIAS, + LIVEKIT_API_SECRET_ALIAS, + DEEPGRAM_API_KEY_ALIAS, + GEMINI_API_KEY_ALIAS, + GOOGLE_API_KEY_ALIAS, + ): + value = context.target_provider_secret_values.get(alias) + if value: + target_environ[alias] = value + self._missing_config = _check_config(context.job, context.target_provider_secret_values) + self._scenario_attempt_counts: dict[str, int] = {} + + async def run(self, scenario: HostedScenario, runtime: EnvironmentRuntime) -> CallOutcome: + if self._missing_config is not None: + # Pre-dial: dialing never starts, so no partial -- and never `WorldUnavailable` (that + # code is reserved by the contract for a world-level capability mismatch, not a + # job-level voice config gap). + raise CallAborted(self._missing_config.message()) + + agent_name = _dispatch_agent_name(runtime) + if agent_name is None: + raise CallAborted( + "voice_dispatch_identity_unavailable: runtime.metadata['livekit_agent_name'] is " + f"not set for world {runtime.world_index}" + ) + + try: + doc = _read_scenario_document(self._context.bundle_dir, scenario.scenario_key) + except _ScenarioDocumentUnavailable as exc: + raise CallAborted(f"voice_scenario_document_unavailable: {exc}") from exc + + scenario_attempt = self._scenario_attempt_counts.get(scenario.scenario_key, 0) + 1 + self._scenario_attempt_counts[scenario.scenario_key] = scenario_attempt + room_name = _room_name( + job_id=self._context.job.job_id, + attempt_number=self._context.attempt_number, + scenario_key=scenario.scenario_key, + scenario_attempt=scenario_attempt, + ) + + raw_timeout = self._context.job.agent.config.get(CALL_TIMEOUT_CONFIG_KEY) + call_timeout_seconds = ( + float(raw_timeout) if isinstance(raw_timeout, (int, float)) else _DEFAULT_CALL_TIMEOUT_SECONDS + ) + run_seconds = ( + call_timeout_seconds + + _CONNECT_TIMEOUT_SECONDS + + _READINESS_TIMEOUT_SECONDS + + _CLEANUP_TIMEOUT_SECONDS + + _RUN_SECONDS_PAD_SECONDS + ) + + spec = _build_spec( + run_id=new_run_id(), + room_name=room_name, + agent_name=agent_name, + doc=doc, + livekit_url=str(self._context.job.agent.config.get(LIVEKIT_URL_CONFIG_KEY)), + call_timeout_seconds=call_timeout_seconds, + run_seconds=run_seconds, + recordings_root=self._context.work_directory / "voice-calls", + ) + + if self._context.evidence_seam is EvidenceSeam.TOOL_TRACE: + endpoint = _find_postgres_endpoint(runtime) + if endpoint is not None: + _clear_tool_trace_calls(endpoint.address) + + started_at = datetime.now(timezone.utc) + outer_timeout = run_seconds + _OUTER_WAIT_FOR_PAD_SECONDS + try: + report = await asyncio.wait_for(self._place_call(spec), timeout=outer_timeout) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError as exc: + raise CallAborted( + "voice_call_runner_timeout: place_call exceeded its outer budget " + f"({outer_timeout:.0f}s)", + partial=self._timing_only_outcome(started_at), + ) from exc + except Exception as exc: # noqa: BLE001 - post-dial machinery failure, never let it escape raw + raise CallAborted( + f"voice_call_runner_crashed: {type(exc).__name__}: {exc}", + partial=self._timing_only_outcome(started_at), + ) from exc + + try: + return await self._translate_report( + report, runtime=runtime, scenario_key=scenario.scenario_key, started_at=started_at + ) + except (CallAborted, WorldUnavailable): + # `_translate_report`'s own typed control-flow (non-completed status, no test case, + # agent-never-joined) -- never re-wrap an intentional abort. + raise + except Exception as exc: # noqa: BLE001 - a transcript/recording read or upload surprise + # must never lose the timing this call already measured (the receipt's `call` field + # must not be null once the call has genuinely started) by escaping run() raw. + raise CallAborted( + f"voice_call_translate_crashed: {type(exc).__name__}: {exc}", + partial=self._timing_only_outcome(started_at), + ) from exc + + def _timing_only_outcome(self, started_at: datetime) -> CallOutcome: + ended_at = datetime.now(timezone.utc) + return CallOutcome( + calls=(), + turns=0, + started_at=format_rfc3339_millis(started_at), + ended_at=format_rfc3339_millis(ended_at), + duration_ms=_duration_ms(started_at, ended_at), + ) + + async def _translate_report( + self, + report: SimulationReport, + *, + runtime: EnvironmentRuntime, + scenario_key: str, + started_at: datetime, + ) -> CallOutcome: + case = report.test_cases[0] if report.test_cases else None + case_started_at = ( + case.started_at if case is not None and case.started_at is not None else started_at + ) + ended_at = ( + case.ended_at + if case is not None and case.ended_at is not None + else datetime.now(timezone.utc) + ) + turns = len(case.result.messages) if case is not None and case.result is not None else 0 + + transcript_artifact: str | None = None + recording_artifacts: list[str] = [] + if case is not None and case.result is not None: + result = case.result + if result.transcript: + transcript_artifact = await self._adapter.upload_artifact( + result.transcript.encode("utf-8"), + kind=ArtifactKind.TRANSCRIPT, + scenario_key=scenario_key, + ) + for path_str, kind in ( + (result.audio_combined_path, ArtifactKind.RECORDING_COMBINED), + (result.audio_stereo_path, ArtifactKind.RECORDING_STEREO), + (result.audio_input_path, ArtifactKind.RECORDING_CUSTOMER), + (result.audio_output_path, ArtifactKind.RECORDING_ASSISTANT), + ): + if not path_str: + continue + path = Path(path_str) + if not path.is_file(): + continue + artifact_id = await self._adapter.upload_artifact( + path.read_bytes(), kind=kind, scenario_key=scenario_key, + ) + if artifact_id is not None: + recording_artifacts.append(artifact_id) + + base = CallOutcome( + calls=(), + turns=turns, + started_at=format_rfc3339_millis(case_started_at), + ended_at=format_rfc3339_millis(ended_at), + duration_ms=_duration_ms(case_started_at, ended_at), + transcript_artifact=transcript_artifact, + recording_artifacts=tuple(recording_artifacts), + ) + + if case is None: + raise CallAborted("voice_call_no_test_case: SimulationReport carried no test case", partial=base) + + if case.status is TestCaseStatus.AGENT_UNAVAILABLE: + # world-handle-interface.md: "the agent never joined" is a WORLD failure, not a + # scenario one -- the agent is part of the world, so the scheduler retires it and + # retries elsewhere. Verified against the engine's own source (engines/livekit.py): + # this status fires ONLY on a readiness-stage timeout with a session already started + # but no target dispatched -- exactly "dispatch fails, agent never joins," never a + # mid-call condition. + reason = case.failure.message if case.failure is not None else "agent_unavailable" + raise WorldUnavailable(f"target agent never joined the room: {reason}") + + # A genuinely silent agent-first call (agent joined, zero conversational turns) reaches + # the real engine (engines/livekit.py::_conversation_outcome) as FAILED with code + # "no_conversation" or "conversation_silence_timeout" and zero messages -- never as a + # COMPLETED case with zero turns (COMPLETED requires >= min_turn_messages AND role + # alternation, so the engine cannot produce that shape). Scoped to zero turns only: a + # short-but-nonzero conversation on either code still failed the completion bar for a real + # reason and must stay a CallAborted below. + is_silent_agent = ( + case.status is TestCaseStatus.FAILED + and turns == 0 + and case.failure is not None + and case.failure.code in _SILENT_AGENT_FAILURE_CODES + ) + + if case.status is not TestCaseStatus.COMPLETED and not is_silent_agent: + reason = case.failure.message if case.failure is not None else case.status.value + raise CallAborted(f"voice_call_not_completed: {case.status.value}: {reason}", partial=base) + + # Never fabricate calls for a call that produced no conversation -- the scheduler's own + # coverage guarantee turns an empty `calls` tuple into evidence_missing/simulator + # regardless of turns (hosted_scheduler.py's own unconditioned-on-turns rule). + calls = () if is_silent_agent else self._collect_calls(runtime) + return CallOutcome( + calls=calls, + turns=base.turns, + started_at=base.started_at, + ended_at=base.ended_at, + duration_ms=base.duration_ms, + transcript_artifact=base.transcript_artifact, + recording_artifacts=base.recording_artifacts, + ) + + def _collect_calls(self, runtime: EnvironmentRuntime) -> tuple[Call, ...]: + seam = self._context.evidence_seam + if seam is EvidenceSeam.HTTP_TOOL: + return _collect_http_tool_calls(runtime) + if seam is EvidenceSeam.TOOL_TRACE: + return _collect_tool_trace_calls(runtime) + # Unrecognized/None (should not happen for a `kind: process` bundle past preflight -- + # bundle_v2.py requires `evidence_seam` whenever `kind is PROCESS` -- but degrading rather + # than crashing keeps this on the scheduler's own evidence_missing path, never a raw + # exception). + return () diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 0d0eb77e..992629c0 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -34,6 +34,7 @@ from . import outbound as ob from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, load_bundle_v2 +from .call_runner import CallRunnerContext, CallRunnerImpl from .hosted_scheduler import ( CallOutcome, CallRunner, @@ -154,6 +155,30 @@ def peek_secret_values(secrets_path: Path) -> tuple[str, ...]: return tuple(str(value) for value in raw.values() if value) +def peek_target_provider_secret_values( + secrets_path: Path, secret_purposes: dict[str, str] +) -> dict[str, str]: + """The same non-destructive, no-unlink read as `peek_secret_values` (same file, same timing + constraint -- called BEFORE `pool.start()`, which is what actually deletes the file), but + ALIAS-preserving and filtered to `purpose: target_provider` -- `peek_secret_values` throws the + alias away, which is fine for outbound redaction (it only needs the raw values) but useless for + the real `CallRunner`, which needs to pick e.g. `LIVEKIT_API_KEY` out of the map by name. Never + fatal: a missing/malformed file just means no target-provider secrets are available yet, + matching `CallRunnerImpl`'s own pre-dial validation (it reports the gap as a typed + `CallAborted`, never crashes on an empty map).""" + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + if not isinstance(raw, dict): + return {} + return { + str(alias): str(value) + for alias, value in raw.items() + if secret_purposes.get(str(alias)) == "target_provider" + } + + # ================================================================================================= # Bundle source -- §2 bundle authoring is not this module's (or built anywhere yet); injectable. # ================================================================================================= @@ -373,7 +398,10 @@ async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Wo # ================================================================================================= -# CallRunner -- the real voice track wires this later. Typed NotWired default only. +# CallRunner -- the real voice track. NotWired stays the fallback for every job whose +# `agent.connector` is not `"livekit"` (absent voice config entirely, or a connector outside the +# LiveKit-dispatched voice path -- vapi/retell/auto) -- by design, not "improved" +# (hosted-execution-seams.md: `"connector": "livekit | vapi | retell | auto"`). # ================================================================================================= @@ -392,6 +420,23 @@ async def run(self, scenario: Scenario, runtime: EnvironmentRuntime) -> CallOutc ) +_LIVEKIT_CONNECTOR = "livekit" + + +def _default_build_call_runner( + adapter: "OutboundAdapter", context: CallRunnerContext +) -> CallRunner: + """The real factory: `NotWiredCallRunner` stays exactly as documented for every connector + outside the LiveKit-dispatched voice path; a `"livekit"` job gets a real `CallRunnerImpl`, + whose OWN pre-dial validation (`call_runner._check_config`) is what surfaces an + incomplete-but-present config as a typed `call_failed`/infrastructure retry -- + `capability_unavailable` stays unreachable from this seam (would require a scheduler edit; + the contract itself calls it "a follow-up, not shipped with this text").""" + if context.job.agent.connector != _LIVEKIT_CONNECTOR: + return NotWiredCallRunner() + return CallRunnerImpl(adapter, context) + + # ================================================================================================= # Scenario pre-allocation -- a thin client against endpoints.scenarios (outbound-channels.md v1.3 # Authentication: bearer + X-Harness-Fence, `{"result": {...}}` envelope, job-scoped idempotent). @@ -1154,11 +1199,14 @@ class HostedEntrypointDeps: build_provider: Callable[[], WorldProvisioner] = field( default=lambda: ProcessRuntimeProvider() ) - # The real call runner needs `OutboundAdapter.upload_artifact` to satisfy the invariant that referenced - # artifacts are uploaded+acked BEFORE the receipt that names them -- the adapter is threaded in - # once `run_job` has built it, rather than the CallRunner reaching for a global. - build_call_runner: Callable[["OutboundAdapter"], CallRunner] = field( - default=lambda adapter: NotWiredCallRunner() + # The real call runner needs `OutboundAdapter.upload_artifact` to satisfy the invariant that + # referenced artifacts are uploaded+acked BEFORE the receipt that names them -- the adapter is + # threaded in once `run_job` has built it, rather than the CallRunner reaching for a global. + # `CallRunnerContext` carries everything else `CallRunnerImpl` needs (job, bundle_dir, + # evidence_seam, the target_provider secret map, attempt_number) that the bare + # `CallRunner.run(scenario, runtime)` protocol has no room for. + build_call_runner: Callable[["OutboundAdapter", CallRunnerContext], CallRunner] = field( + default=lambda adapter, context: _default_build_call_runner(adapter, context) ) build_world_factory: Callable[[Path], WorldFactory] = field(default=ProcessWorldFactory) retry_policy: Callable[[], ob.RetryPolicy] = field(default=lambda: ob.RetryPolicy()) @@ -1190,6 +1238,11 @@ def build_scenarios_client( def peek_secret_values(self) -> tuple[str, ...]: return peek_secret_values(self.secrets_path) + def peek_target_provider_secret_values( + self, secret_purposes: dict[str, str] + ) -> dict[str, str]: + return peek_target_provider_secret_values(self.secrets_path, secret_purposes) + # ================================================================================================= # Scenario-entry validation at fetch (defense against karthik-integration-changes.md K1): the @@ -1447,6 +1500,13 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m job_seed = job.seed if job.seed is not None else 0 parallelism = resolve_parallelism(job) secret_purposes = job_secret_purposes(job) + # `ProcessRuntimeProvider` deletes `secrets.json` on its FIRST `provision()` call, inside + # `pool.start()` below -- this capture must happen (and does: `job` is only just now + # available, but `pool.start()` is still ~50 lines further down) strictly BEFORE that + # point, same constraint `deps.peek_secret_values()` already satisfies for redaction, + # above at adapter construction. Alias-preserving so `CallRunnerImpl` can pick e.g. + # `LIVEKIT_API_KEY` out of the map by name. + target_provider_secret_values = deps.peek_target_provider_secret_values(secret_purposes) adapter.configure_artifacts(job.artifacts) # level table + budget, now that job.json is known. adapter.stage_changed(HarnessStage.VALIDATING_ENVIRONMENT) @@ -1659,7 +1719,15 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m # 5/6. Scheduler wiring. adapter.stage_changed(HarnessStage.RUNNING) await adapter.aflush_events() - call_runner = deps.build_call_runner(adapter) + call_runner_context = CallRunnerContext( + job=job, + bundle_dir=bundle_dir, + work_directory=work_directory, + evidence_seam=manifest.runtime.evidence_seam, + target_provider_secret_values=target_provider_secret_values, + attempt_number=capabilities.attempt_number, + ) + call_runner = deps.build_call_runner(adapter, call_runner_context) scheduler = HostedScheduler( pool=pool, world_factory=world_factory, call_runner=call_runner, outbound=adapter, job_seed=job_seed, cancel_requested=cancel_requested, diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py new file mode 100644 index 00000000..dc1a8417 --- /dev/null +++ b/tests/harness/test_call_runner.py @@ -0,0 +1,799 @@ +"""`call_runner.py` against in-memory fakes and scripted `SimulationReport`s -- no real LiveKit +call, no real postgres, no monkeypatching LiveKit internals. Matches this repo's own convention +(`test_hosted_entrypoint.py`'s docstring): `asyncio.run` drives every `async def` seam directly. + +The test seam is `CallRunnerImpl`'s own boundary: the injectable `place_call(spec) -> +SimulationReport` callable named by the brief. Real production code (`_default_place_call`) +builds the same `SimulationSpec`/`SimulationRunner().run` pair; these tests never construct or +touch a real `SimulationRunner`, `rtc.Room`, or `AgentSession`. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from fi.alk.harness import call_runner as cr +from fi.alk.harness.bundle_v2 import EvidenceSeam +from fi.alk.harness.hosted_scheduler import CallAborted, CallOutcome +from fi.alk.harness.job import ( + AgentConnection, + ExecutionMode, + HarnessJob, + RepositorySource, + SourceKind, + SourceVisibility, +) +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeEndpoint, RuntimeState +from fi.alk.harness.world.errors import WorldUnavailable +from fi.alk.harness.world.runtime import Call +from fi.simulate.artifacts import ArtifactManifest +from fi.simulate.runtime.failures import FailureStage, SimulationFailure +from fi.simulate.runtime.report import SimulationReport, SimulationTestCaseResult +from fi.simulate.runtime.run import RunStatus +from fi.simulate.runtime.run import TestCaseStatus as CaseStatus +from fi.simulate.simulation.models import Persona as SimPersona +from fi.simulate.simulation.models import TestCaseResult as SimTestCaseResult + +LIVEKIT_API_KEY = "LIVEKIT_API_KEY" +LIVEKIT_API_SECRET = "LIVEKIT_API_SECRET" +DEEPGRAM_API_KEY = "DEEPGRAM_API_KEY" +GEMINI_API_KEY = "GEMINI_API_KEY" + +_ALL_SECRETS = { + LIVEKIT_API_KEY: "lk-key", + LIVEKIT_API_SECRET: "lk-secret", + DEEPGRAM_API_KEY: "dg-key", + GEMINI_API_KEY: "gm-key", +} +_ALL_CONFIG = {cr.LIVEKIT_URL_CONFIG_KEY: "wss://example.livekit.cloud"} + + +# ================================================================================================= +# Fixtures -- self-contained (this file touches nothing outside itself + call_runner.py). +# ================================================================================================= + + +def _job(*, connector: str = "livekit", config: dict[str, Any] | None = None) -> HarnessJob: + return HarnessJob( + job_id="job-abcdef12-xyz", + run_id="run-1", + execution=ExecutionMode.HOSTED, + source=RepositorySource( + kind=SourceKind.GITHUB, repository="org/repo", visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + ), + agent=AgentConnection(connector=connector, config=config or {}), + scenario_count=1, + seed=1, + ) + + +def _runtime( + *, + metadata: dict[str, Any] | None = None, + endpoints: dict[str, RuntimeEndpoint] | None = None, + world_index: int = 0, +) -> EnvironmentRuntime: + return EnvironmentRuntime( + runtime_id="rt-1", + world_index=world_index, + bundle_digest="sha256:" + "0" * 64, + state=RuntimeState.READY, + endpoints=endpoints or {}, + metadata=metadata or {}, + ) + + +def _postgres_endpoint(*, address: str = "postgresql://harness:pw@localhost:15001/w0") -> RuntimeEndpoint: + return RuntimeEndpoint( + capability="database", protocol="postgres", address=address, configuration_name="DATABASE_URL", + ) + + +def _context( + *, + tmp_path: Path, + connector: str = "livekit", + config: dict[str, Any] | None = None, + secrets: dict[str, str] | None = None, + evidence_seam: EvidenceSeam | None = EvidenceSeam.HTTP_TOOL, + attempt_number: int = 1, +) -> tuple[HarnessJob, cr.CallRunnerContext]: + job = _job(connector=connector, config=config if config is not None else dict(_ALL_CONFIG)) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir(parents=True, exist_ok=True) + context = cr.CallRunnerContext( + job=job, + bundle_dir=bundle_dir, + work_directory=tmp_path / "work", + evidence_seam=evidence_seam, + target_provider_secret_values=secrets if secrets is not None else dict(_ALL_SECRETS), + attempt_number=attempt_number, + ) + return job, context + + +def _write_scenario_doc( + bundle_dir: Path, + *, + scenario_key: str, + folder_name: str | None = None, + instruction: str = "Cancel order #42.", + persona: dict[str, Any] | None = None, + fixture: dict[str, Any] | None = None, + tests: str = "", +) -> None: + folder = bundle_dir / "scenarios" / (folder_name or scenario_key) + folder.mkdir(parents=True, exist_ok=True) + body = { + "scenario_key": scenario_key, + "scenario_id": "", + "sub_goals": [], + "instruction": instruction, + "persona": persona, + "fixture": fixture or {}, + "tests": tests, + } + (folder / "scenario.json").write_text(json.dumps(body), encoding="utf-8") + + +@dataclass +class _FakeScenario: + """`CallRunnerImpl.run` reads only `.scenario_key` off the scheduler's `Scenario` protocol.""" + + scenario_key: str + scenario_id: str = "" + + +@dataclass +class FakeAdapter: + """Narrow `ArtifactUploader` fake -- records every call, returns a real `sha256:` id + unless the kind is in `refuse_kinds` (mirrors the real adapter's null-not-crash contract).""" + + refuse_kinds: frozenset = field(default_factory=frozenset) + uploads: list[tuple[Any, str | None, bytes]] = field(default_factory=list) + + async def upload_artifact(self, data: bytes, *, kind, scenario_key=None, deadline=None) -> str | None: + if kind in self.refuse_kinds: + return None + import hashlib + + digest = hashlib.sha256(data).hexdigest() + self.uploads.append((kind, scenario_key, data)) + return f"sha256:{digest}" + + +def _persona() -> SimPersona: + return SimPersona(persona={"name": "customer"}, situation="s", outcome="o") + + +def _report( + *, + status: RunStatus = RunStatus.COMPLETED, + case_status: CaseStatus | None = CaseStatus.COMPLETED, + transcript: str = "hello there", + messages: list[dict[str, str]] | None = None, + failure: SimulationFailure | None = None, + started_at: datetime | None = None, + ended_at: datetime | None = None, + no_cases: bool = False, + run_id: str = "sim-run-1", +) -> SimulationReport: + messages = messages if messages is not None else [{"role": "user", "content": "hi"}] + started_at = started_at or datetime.now(timezone.utc) + ended_at = ended_at or (started_at + timedelta(seconds=30)) + cases: list[SimulationTestCaseResult] = [] + if not no_cases: + assert case_status is not None + result = SimTestCaseResult(persona=_persona(), transcript=transcript, messages=messages) + cases.append( + SimulationTestCaseResult( + test_case_id="tc-1", status=case_status, persona=_persona(), result=result, + failure=failure, started_at=started_at, ended_at=ended_at, + ) + ) + return SimulationReport( + run_id=run_id, spec_hash="hash", status=status, started_at=started_at, ended_at=ended_at, + test_cases=cases, artifacts=ArtifactManifest(run_id=run_id), + ) + + +def _run(runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + return asyncio.run(runner.run(scenario, runtime)) + + +def _run_expect_abort( + runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime +) -> CallAborted: + try: + asyncio.run(runner.run(scenario, runtime)) + except CallAborted as exc: + return exc + raise AssertionError("expected CallAborted, nothing was raised") + + +def _run_expect_world_unavailable( + runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime +) -> WorldUnavailable: + try: + asyncio.run(runner.run(scenario, runtime)) + except WorldUnavailable as exc: + return exc + raise AssertionError("expected WorldUnavailable, nothing was raised") + + +# ================================================================================================= +# Room naming (deterministic scheme, pinned verbatim by this file). WHY only a prefix at the wire: +# engines/livekit.py::_resolve_room_name appends its own `-{invocation_id}-{test_case_id[-12:]}` +# suffix in managed room_mode unless `room_name_verbatim` is set (this runner does not set it), so +# the pinned string below is the deterministic PREFIX every dialed room carries, not the full +# on-the-wire room name. +# ================================================================================================= + + +def test_room_name_matches_the_pinned_deterministic_scheme() -> None: + name = cr._room_name(job_id="abcdef1234567890", attempt_number=2, scenario_key="cancel-order", scenario_attempt=3) + assert name == "harness-abcdef12-a2-cancel-order-s3" + + +def test_room_name_uses_only_the_first_eight_chars_of_job_id() -> None: + short = cr._room_name(job_id="ab", attempt_number=1, scenario_key="k", scenario_attempt=1) + assert short == "harness-ab-a1-k-s1" + + +# ================================================================================================= +# Pre-dial validation. +# ================================================================================================= + + +def test_missing_target_provider_secrets_aborts_pre_dial_without_calling_place_call(tmp_path: Path) -> None: + called = False + + async def place_call(spec): + nonlocal called + called = True + raise AssertionError("place_call must never be reached") + + _job_obj, context = _context(tmp_path=tmp_path, secrets={}) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is None + assert str(exc).startswith("voice_capability_unavailable: missing") + assert LIVEKIT_API_KEY in str(exc) + assert not called + + +def test_missing_llm_credential_names_the_either_or_pair(tmp_path: Path) -> None: + secrets = dict(_ALL_SECRETS) + del secrets[GEMINI_API_KEY] + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "GEMINI_API_KEY_or_GOOGLE_API_KEY" in str(exc) + + +def test_google_api_key_alone_satisfies_the_llm_credential_check(tmp_path: Path) -> None: + secrets = dict(_ALL_SECRETS) + del secrets[GEMINI_API_KEY] + secrets["GOOGLE_API_KEY"] = "g-key" + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + assert runner._missing_config is None + + +def test_missing_livekit_url_config_aborts_pre_dial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path, config={}) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "config=livekit_url" in str(exc) + + +def test_missing_dispatch_identity_metadata_aborts_pre_dial(tmp_path: Path) -> None: + """Verified finding: `EnvironmentRuntime.metadata` is always `{}` in this worktree's HEAD -- + this is the realistic default a real hosted run hits today (CONTRACT NOTE 3).""" + called = False + + async def place_call(spec): + nonlocal called + called = True + raise AssertionError("place_call must never be reached") + + _job_obj, context = _context(tmp_path=tmp_path) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={})) + assert "voice_dispatch_identity_unavailable" in str(exc) + assert "livekit_agent_name" in str(exc) + assert not called + + +def test_missing_scenario_document_aborts_pre_dial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort( + runner, _FakeScenario("no-such-key"), _runtime(metadata={"livekit_agent_name": "agent-w0"}) + ) + assert "voice_scenario_document_unavailable" in str(exc) + + +def test_scenario_document_matched_by_scenario_key_field_not_folder_name(tmp_path: Path) -> None: + """scenario_source.py's own convention: `scenario_key` is a field INSIDE scenario.json, not + necessarily the folder name -- this runner must match the same way, not assume they agree.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="the-real-key", folder_name="some-other-folder-name") + + captured: dict[str, Any] = {} + + async def place_call(spec): + captured["spec"] = spec + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + _run(runner, _FakeScenario("the-real-key"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert captured["spec"] is not None + + +# ================================================================================================= +# Happy path: COMPLETED -> real CallOutcome, artifacts uploaded, dispatch/room wiring correct. +# ================================================================================================= + + +def test_completed_call_uploads_transcript_and_returns_populated_outcome(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=EvidenceSeam.HTTP_TOOL) + _write_scenario_doc(context.bundle_dir, scenario_key="k1", instruction="Cancel order #42.") + + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended = started + timedelta(seconds=45) + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}] + + async def place_call(spec): + return _report(transcript="hi\nhello", messages=messages, started_at=started, ended_at=ended) + + adapter = FakeAdapter() + runner = cr.CallRunnerImpl(adapter, context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 2 + assert outcome.duration_ms == 45_000 + assert outcome.transcript_artifact is not None + assert outcome.transcript_artifact.startswith("sha256:") + assert outcome.calls == () # http_tool: STOPPED, always zero -- CONTRACT NOTE 1 + assert len(adapter.uploads) == 1 + + +def test_dispatch_agent_name_and_livekit_url_flow_into_the_built_spec(tmp_path: Path) -> None: + _job_obj, context = _context( + tmp_path=tmp_path, config={cr.LIVEKIT_URL_CONFIG_KEY: "wss://custom.livekit.cloud"}, + ) + _write_scenario_doc(context.bundle_dir, scenario_key="k1", instruction="Do the thing.") + + captured: dict[str, Any] = {} + + async def place_call(spec): + captured["spec"] = spec + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + + spec = captured["spec"] + agent_definition = spec.environment.config["agent_definition"] + livekit_runtime = spec.environment.config["livekit_runtime"] + assert agent_definition["agent_name"] == "agent-w0" + assert agent_definition["system_prompt"] == "Do the thing." + # WHY prefix-match, not exact-match: this asserts what THIS runner puts on the spec, which is + # exactly the pinned deterministic scheme -- but engines/livekit.py::_resolve_room_name appends + # its own suffix in managed room_mode before the room is actually dialed, so a reader must not + # take this string as the full on-the-wire room name. + assert livekit_runtime["room_name"].startswith("harness-job-abcd-a1-k1-s1") + assert livekit_runtime["url"] == "wss://custom.livekit.cloud/" + + +def test_scenario_attempt_counter_increments_per_scenario_key_across_retries(tmp_path: Path) -> None: + """The scheduler retries the SAME scenario_key (e.g. after evidence_missing) -- successive + `run()` calls for one key must get distinct room names, or a LiveKit room collision follows.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + _write_scenario_doc(context.bundle_dir, scenario_key="k2") + + rooms: list[str] = [] + + async def place_call(spec): + rooms.append(spec.environment.config["livekit_runtime"]["room_name"]) + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + runtime = _runtime(metadata={"livekit_agent_name": "agent-w0"}) + _run(runner, _FakeScenario("k1"), runtime) + _run(runner, _FakeScenario("k2"), runtime) + _run(runner, _FakeScenario("k1"), runtime) + + assert rooms[0].endswith("-k1-s1") + assert rooms[1].endswith("-k2-s1") + assert rooms[2].endswith("-k1-s2") + + +# ================================================================================================= +# Failure semantics -- the three cases the brief pins. +# ================================================================================================= + + +def test_agent_unavailable_status_raises_world_unavailable(tmp_path: Path) -> None: + """Verified against engines/livekit.py: AGENT_UNAVAILABLE fires ONLY on a readiness-stage + timeout with a session started but no target dispatched -- exactly "dispatch fails, agent + never joins," matching the brief's explicit WorldUnavailable case.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.READINESS, code="agent_unavailable", + message="Target agent did not become ready", + ) + return _report(case_status=CaseStatus.AGENT_UNAVAILABLE, failure=failure) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_world_unavailable( + runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"}) + ) + assert "Target agent did not become ready" in str(exc) + + +def test_non_completed_status_raises_call_aborted_with_partial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended = started + timedelta(seconds=12) + + async def place_call(spec): + failure = SimulationFailure(stage=FailureStage.RUNNING, code="case_execution_error", message="boom") + return _report( + case_status=CaseStatus.FAILED, failure=failure, started_at=started, ended_at=ended, + transcript="", messages=[], + ) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.duration_ms == 12_000 + assert exc.partial.calls == () + assert "boom" in str(exc) + + +def test_no_test_cases_in_report_raises_call_aborted_with_timing_only_partial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + return _report(status=RunStatus.TIMED_OUT, no_cases=True) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.turns == 0 + assert exc.partial.calls == () + + +def test_place_call_exception_raises_call_aborted_with_timing_partial_never_raw(tmp_path: Path) -> None: + """world-handle-interface.md's partial-call rule: a generic exception must never lose timing + (the brief: "never let a raw exception escape post-dial").""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + raise RuntimeError("engine exploded") + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.started_at is not None + assert exc.partial.ended_at is not None + assert "engine exploded" in str(exc) + + +def test_translate_report_upload_failure_raises_call_aborted_with_timing_partial_never_raw( + tmp_path: Path, +) -> None: + """The same partial-call rule as `place_call` failing above, but for a failure INSIDE + `_translate_report` itself (a transcript/recording read or an `upload_artifact` surprise) -- + this must also never escape `run()` raw and lose the timing the call already measured.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + class RaisingAdapter: + async def upload_artifact(self, data, *, kind, scenario_key=None, deadline=None): + raise RuntimeError("upload exploded") + + async def place_call(spec): + return _report( + transcript="hi", + messages=[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}], + ) + + runner = cr.CallRunnerImpl(RaisingAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.started_at is not None + assert exc.partial.ended_at is not None + assert exc.partial.calls == () + assert "upload exploded" in str(exc) + + +def test_place_call_outer_timeout_raises_call_aborted_with_timing_partial( + tmp_path: Path, monkeypatch +) -> None: + """Forces the runner-owned `asyncio.wait_for` to actually fire (not just the SDK's own + internal one) by shrinking every phase-overhead constant to a few milliseconds -- avoids a + multi-minute real sleep in the test suite while still exercising the real timeout code path.""" + monkeypatch.setattr(cr, "_CONNECT_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_READINESS_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_CLEANUP_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_RUN_SECONDS_PAD_SECONDS", 0.01) + monkeypatch.setattr(cr, "_OUTER_WAIT_FOR_PAD_SECONDS", 0.01) + + _job_obj, context = _context( + tmp_path=tmp_path, config={cr.LIVEKIT_URL_CONFIG_KEY: "wss://x", cr.CALL_TIMEOUT_CONFIG_KEY: 0.01}, + ) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + await asyncio.Event().wait() # never completes on its own + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "voice_call_runner_timeout" in str(exc) + assert exc.partial is not None + assert exc.partial.calls == () + + +def test_silent_agent_real_engine_shape_returns_normal_outcome_with_empty_calls(tmp_path: Path) -> None: + """Pins the shape the real engine actually produces for a silent agent-first call + (engines/livekit.py::_conversation_outcome): status FAILED, code "no_conversation", zero + messages -- never a COMPLETED case with zero turns (COMPLETED requires >= min_turn_messages + and role alternation, a shape the engine cannot produce for a silent call). Must still surface + as a NORMAL CallOutcome with the real (zero) turn count, never a WorldUnavailable/CallAborted, + letting the scheduler's own coverage guarantee turn it into evidence_missing.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="no_conversation", + message="No conversation turns were committed before the inactivity deadline", + retryable=True, + ) + return _report(case_status=CaseStatus.FAILED, failure=failure, transcript="", messages=[]) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 0 + assert outcome.calls == () + assert outcome.transcript_artifact is None # empty transcript -- never uploaded + + +def test_silent_agent_conversation_silence_timeout_code_also_returns_normal_outcome(tmp_path: Path) -> None: + """The engine's other zero-turn silent code (the agent-first silence watchdog firing before + any turn ever lands) must map the same way as "no_conversation" above.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="conversation_silence_timeout", + message="Agent-first conversation stalled after it began", retryable=True, + ) + return _report(case_status=CaseStatus.FAILED, failure=failure, transcript="", messages=[]) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 0 + assert outcome.calls == () + + +def test_silent_agent_mapping_is_scoped_to_zero_turns_only(tmp_path: Path) -> None: + """A short-but-nonzero conversation carrying the same failure code must NOT be laundered into + a normal outcome -- only a genuinely zero-turn silent call qualifies; this stays a CallAborted + exactly like any other non-completed status.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}] + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="conversation_silence_timeout", + message="Agent-first conversation stalled after it began", retryable=True, + ) + return _report( + case_status=CaseStatus.FAILED, failure=failure, transcript="hi\nhello", messages=messages, + ) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.turns == 2 + + +# ================================================================================================= +# Evidence collection. +# ================================================================================================= + + +def test_http_tool_seam_always_returns_no_calls() -> None: + """CONTRACT NOTE 1: STOPPED, verified -- no invented capture proxy.""" + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + assert cr._collect_http_tool_calls(runtime) == () + + +def test_find_postgres_endpoint_matches_by_protocol_not_a_fixed_slug_name() -> None: + """Corrects the brief's literal `runtime.endpoints["database"]` wording: capability slugs are + bundle-author-chosen (verified against `build_endpoints`/`_find_postgres_endpoint` in + hosted_entrypoint.py) -- a bundle naming its slug anything else must still resolve.""" + endpoint = _postgres_endpoint() + runtime = _runtime(endpoints={"orders_store": endpoint}) + found = cr._find_postgres_endpoint(runtime) + assert found is endpoint + + +def test_find_postgres_endpoint_returns_none_when_absent() -> None: + runtime = _runtime(endpoints={"queue": RuntimeEndpoint(capability="queue", protocol="amqp", address="amqp://x")}) + assert cr._find_postgres_endpoint(runtime) is None + + +class _FakeCursor: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self.description = [(name,) for name in columns] + + def fetchall(self) -> list[tuple[Any, ...]]: + return self._rows + + +class _FakeConnection: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self._columns = columns + self.executed: list[str] = [] + + def execute(self, statement: str, params: Any = None) -> _FakeCursor: + self.executed.append(statement) + return _FakeCursor(self._rows, self._columns) + + def __enter__(self) -> "_FakeConnection": + return self + + def __exit__(self, *exc_info: Any) -> bool: + return False + + +class _FakePsycopg: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self._columns = columns + self.connections: list[_FakeConnection] = [] + + def connect(self, dsn: str, **kwargs: Any) -> _FakeConnection: + connection = _FakeConnection(self._rows, self._columns) + self.connections.append(connection) + return connection + + +class _RaisingPsycopg: + class Error(Exception): + pass + + def connect(self, dsn: str, **kwargs: Any) -> Any: + raise self.Error("connection refused") + + +def test_tool_trace_translates_rows_and_applies_v1_refused_rule(monkeypatch) -> None: + """world-handle-interface.md V1 rule: `refused = not ok` (a trace cannot distinguish refusal + from crash).""" + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [ + ("lookup_order", {"id": "1"}, {"status": "shipped"}, True, "", 100.5), + ("cancel_order", {"id": "2"}, None, False, "not found", 101.0), + ] + fake = _FakePsycopg(rows, columns) + monkeypatch.setitem(sys.modules, "psycopg", fake) + + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + + assert calls == ( + Call(name="lookup_order", arguments={"id": "1"}, result={"status": "shipped"}, ok=True, error="", refused=False, at=100.5), + Call(name="cancel_order", arguments={"id": "2"}, result=None, ok=False, error="not found", refused=True, at=101.0), + ) + + +def test_tool_trace_read_failure_degrades_to_no_calls_never_crashes(monkeypatch) -> None: + """No producer exists yet (CONTRACT NOTE 2) -- a missing table / connection failure must + degrade to `()`, never raise past this function.""" + monkeypatch.setitem(sys.modules, "psycopg", _RaisingPsycopg()) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + assert cr._collect_tool_trace_calls(runtime) == () + + +def test_tool_trace_missing_endpoint_degrades_to_no_calls() -> None: + runtime = _runtime(endpoints={}) + assert cr._collect_tool_trace_calls(runtime) == () + + +def test_tool_trace_result_string_form_is_truncated_at_2000_chars(monkeypatch) -> None: + long_string = "x" * 3000 + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("t", {}, long_string, True, "", 1.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + assert len(calls) == 1 + assert len(calls[0].result) == 2000 + + +def test_tool_trace_parsed_json_result_is_not_truncated(monkeypatch) -> None: + """Per world-handle-interface.md: "result — parsed JSON where the source captured JSON, else + a string; both result (string form) and error truncated at 2,000 chars" -- truncation applies + to the STRING form only.""" + big_list = list(range(3000)) + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("t", {}, big_list, True, "", 1.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + assert calls[0].result == big_list + + +def test_tool_trace_clear_is_best_effort_and_never_raises(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "psycopg", _RaisingPsycopg()) + cr._clear_tool_trace_calls("postgresql://x/y") # must not raise + + +def test_completed_call_with_tool_trace_seam_collects_evidence(tmp_path: Path, monkeypatch) -> None: + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("do_thing", {}, "done", True, "", 5.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=EvidenceSeam.TOOL_TRACE) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + return _report() + + runtime = _runtime( + metadata={"livekit_agent_name": "agent-w0"}, endpoints={"database": _postgres_endpoint()}, + ) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), runtime) + assert len(outcome.calls) == 1 + assert outcome.calls[0].name == "do_thing" + + +# ================================================================================================= +# Credential export (WHY: the LiveKit engine reads these via ambient os.environ, not spec fields). +# ================================================================================================= + + +def test_construction_exports_target_provider_secrets_to_environ_once(tmp_path: Path) -> None: + fake_environ: dict[str, str] = {} + _job_obj, context = _context(tmp_path=tmp_path) + cr.CallRunnerImpl(FakeAdapter(), context, environ=fake_environ) + assert fake_environ[LIVEKIT_API_KEY] == "lk-key" + assert fake_environ[LIVEKIT_API_SECRET] == "lk-secret" + assert fake_environ[DEEPGRAM_API_KEY] == "dg-key" + assert fake_environ[GEMINI_API_KEY] == "gm-key" + + +def test_construction_never_exports_secrets_outside_the_target_provider_map(tmp_path: Path) -> None: + fake_environ: dict[str, str] = {} + secrets = dict(_ALL_SECRETS) + secrets["UNRELATED_ALIAS"] = "should-not-export" + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + cr.CallRunnerImpl(FakeAdapter(), context, environ=fake_environ) + assert "UNRELATED_ALIAS" not in fake_environ diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index 1de92836..ba41230e 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -28,6 +28,7 @@ from fi.alk.harness.bundle_v2 import ( BUNDLE_V2_SCHEMA_VERSION, EnvironmentBundleV2, + EvidenceSeam, ManagedEngine, compute_inputs_digest, seal_bundle_v2, @@ -572,8 +573,11 @@ def _build_harness( holder: dict[str, he.OutboundAdapter] = {} - def build_call_runner(adapter: he.OutboundAdapter) -> FakeCallRunner: + def build_call_runner( + adapter: he.OutboundAdapter, context: he.CallRunnerContext + ) -> FakeCallRunner: holder["adapter"] = adapter + holder["call_runner_context"] = context return FakeCallRunner( adapter, cancel_path=cancel_path, cancel_on_scenario=cancel_on_scenario, cancel_reason=cancel_reason, @@ -687,6 +691,136 @@ def test_peek_secret_values_missing_file_is_empty() -> None: assert he.peek_secret_values(Path("/nonexistent/does-not-exist.json")) == () +def test_peek_target_provider_secret_values_filters_by_purpose_and_keeps_the_alias() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p14-secrets-")) + path = tmp / "secrets.json" + path.write_text( + json.dumps( + { + "LIVEKIT_API_KEY": "lk-secret", + "GITHUB_INSTALLATION_TOKEN": "gh-secret", + } + ), + encoding="utf-8", + ) + values = he.peek_target_provider_secret_values( + path, + {"LIVEKIT_API_KEY": "target_provider", "GITHUB_INSTALLATION_TOKEN": "source_checkout"}, + ) + assert values == {"LIVEKIT_API_KEY": "lk-secret"} + assert path.exists() # non-destructive read, same timing contract as peek_secret_values. + + +def test_peek_target_provider_secret_values_missing_file_is_empty() -> None: + assert ( + he.peek_target_provider_secret_values( + Path("/nonexistent/does-not-exist.json"), {"LIVEKIT_API_KEY": "target_provider"} + ) + == {} + ) + + +def test_peek_target_provider_secret_values_drops_an_alias_with_no_purpose_entry() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p14-secrets-")) + path = tmp / "secrets.json" + path.write_text(json.dumps({"UNKNOWN_ALIAS": "value"}), encoding="utf-8") + assert he.peek_target_provider_secret_values(path, {}) == {} + + +# ================================================================================================= +# CallRunner wiring (p14): the extended build_call_runner(adapter, context) seam, and the +# NotWired-stays-the-fallback / real-CallRunnerImpl split on `agent.connector`. +# ================================================================================================= + + +def _call_runner_context( + *, job: HarnessJob | None = None, evidence_seam: Any = EvidenceSeam.HTTP_TOOL +) -> he.CallRunnerContext: + return he.CallRunnerContext( + job=job or _job(), + bundle_dir=Path("/nonexistent/bundle"), + work_directory=Path("/nonexistent/work"), + evidence_seam=evidence_seam, + target_provider_secret_values={}, + attempt_number=1, + ) + + +def test_default_build_call_runner_returns_notwired_for_a_non_livekit_connector() -> None: + # `_job()`'s own default is `connector="vapi"` -- out of this worker's mission, by design. + runner = he._default_build_call_runner(mock.Mock(), _call_runner_context(job=_job(connector="vapi"))) + assert isinstance(runner, he.NotWiredCallRunner) + + +def test_default_build_call_runner_returns_notwired_for_retell_and_auto_too() -> None: + for connector in ("retell", "auto"): + runner = he._default_build_call_runner( + mock.Mock(), _call_runner_context(job=_job(connector=connector)) + ) + assert isinstance(runner, he.NotWiredCallRunner) + + +def test_default_build_call_runner_returns_a_real_call_runner_impl_for_livekit() -> None: + runner = he._default_build_call_runner(mock.Mock(), _call_runner_context(job=_job(connector="livekit"))) + assert isinstance(runner, he.CallRunnerImpl) + + +def test_call_runner_context_is_threaded_with_real_job_bundle_secrets_and_evidence_seam() -> None: + """End-to-end through the real `run_job` wiring point (~line 1728): `secret_purposes = + job_secret_purposes(job)` runs, `deps.peek_target_provider_secret_values` captures the alias + map BEFORE `pool.start()` deletes `secrets.json`, and the resulting `CallRunnerContext` reaches + whatever factory `deps.build_call_runner` names -- verified by capturing it, not by asserting + on `CallRunnerImpl` internals (that belongs to test_call_runner.py). + + `SecretDeletingProvisioner` mirrors the REAL `ProcessRuntimeProvider`'s own lifetime rule + (process_runtime.py:3535-3544): `secrets.json` is deleted on the provisioner's FIRST + `provision()` call, which `pool.start()` awaits synchronously. Without this, `FakeProvisioner` + never touches the file at all, and a capture that happened AFTER `pool.start()` instead of + before would still see the intact secrets file and still pass -- the deletion is what makes + the capture's timing actually load-bearing here, not just documented in a comment.""" + + class SecretDeletingProvisioner(FakeProvisioner): + def __init__(self, *args: Any, secrets_path: Path, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._secrets_path = secrets_path + + async def provision(self, *args: Any, **kwargs: Any) -> list[EnvironmentRuntime]: + runtimes = await super().provision(*args, **kwargs) + self._secrets_path.unlink(missing_ok=True) + return runtimes + + harness = _build_harness(scenarios=[]) + harness.deps.secrets_path.parent.mkdir(parents=True, exist_ok=True) + harness.deps.secrets_path.write_text( + json.dumps({TARGET_PROVIDER_ALIAS: "lk-secret-value"}), encoding="utf-8" + ) + harness.deps.build_provider = lambda: SecretDeletingProvisioner( + instances=1, secrets_path=harness.deps.secrets_path + ) + + captured: dict[str, he.CallRunnerContext] = {} + + def build_call_runner(adapter: he.OutboundAdapter, context: he.CallRunnerContext) -> FakeCallRunner: + captured["context"] = context + return FakeCallRunner(adapter, cancel_path=harness.deps.cancel_path) + + harness.deps.build_call_runner = build_call_runner + code = _run(harness) + assert code == he.EXIT_OK + + context = captured["context"] + assert context.job.job_id == "job-1" + assert context.bundle_dir == harness.bundle_dir + # `_base_manifest_body()` (this file's own bundle fixture) declares `evidence_seam: "http_tool"`. + assert context.evidence_seam is EvidenceSeam.HTTP_TOOL + assert context.attempt_number == 1 + # The load-bearing assertion: the secrets file is genuinely gone by the time `provision()` + # (inside `pool.start()`) returns -- if the capture had happened AFTER `pool.start()` instead + # of before, this map would be empty, not the real alias -> value pair. + assert context.target_provider_secret_values == {TARGET_PROVIDER_ALIAS: "lk-secret-value"} + assert not harness.deps.secrets_path.exists() + + def test_row_counts_for_capability_returns_the_matching_store() -> None: build_output = {"stores": [{"capability": "database", "row_counts": {"riders": 3}}]} assert he.row_counts_for_capability(build_output, "database") == {"riders": 3} @@ -1582,7 +1716,7 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> Call scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] harness = _build_harness(scenarios=scenarios, instances=1) - harness.deps.build_call_runner = lambda adapter: ChattyCallRunner(adapter, log_count=260) + harness.deps.build_call_runner = lambda adapter, context: ChattyCallRunner(adapter, log_count=260) code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) assert code == he.EXIT_OK @@ -1619,7 +1753,7 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> Call scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] harness = _build_harness(scenarios=scenarios, instances=1) - harness.deps.build_call_runner = lambda adapter: AbortingCallRunner() + harness.deps.build_call_runner = lambda adapter, context: AbortingCallRunner() code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) assert code == he.EXIT_OK @@ -1708,7 +1842,7 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> Call scenarios=scenarios, instances=1, artifacts=HarnessArtifactPolicy(level=ArtifactLevel.TRACES), ) - harness.deps.build_call_runner = lambda adapter: RecordingCallRunner(adapter) + harness.deps.build_call_runner = lambda adapter, context: RecordingCallRunner(adapter) code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) assert code == he.EXIT_OK @@ -2173,7 +2307,7 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> Call harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) transport = OrderTrackingTransport() harness.deps.build_transport = lambda: transport - harness.deps.build_call_runner = lambda adapter: ChattyCancelingCallRunner( + harness.deps.build_call_runner = lambda adapter, context: ChattyCancelingCallRunner( adapter, cancel_path=harness.deps.cancel_path, cancel_on_scenario="first", chatter_count=260, ) @@ -2682,6 +2816,13 @@ async def scenario() -> None: test_job_secret_purposes_maps_alias_to_purpose, test_peek_secret_values_reads_without_deleting, test_peek_secret_values_missing_file_is_empty, + test_peek_target_provider_secret_values_filters_by_purpose_and_keeps_the_alias, + test_peek_target_provider_secret_values_missing_file_is_empty, + test_peek_target_provider_secret_values_drops_an_alias_with_no_purpose_entry, + test_default_build_call_runner_returns_notwired_for_a_non_livekit_connector, + test_default_build_call_runner_returns_notwired_for_retell_and_auto_too, + test_default_build_call_runner_returns_a_real_call_runner_impl_for_livekit, + test_call_runner_context_is_threaded_with_real_job_bundle_secrets_and_evidence_seam, test_row_counts_for_capability_returns_the_matching_store, test_row_counts_for_capability_raises_when_the_capability_is_absent, test_cancel_state_reads_reason_from_file, From ff72caa0da791783a37551f925b998b403c0b1b4 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Wed, 26 Aug 2026 21:14:43 +0530 Subject: [PATCH 18/20] feat(harness): surface the voice dispatch identity on runtime metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The call runner reads runtime.metadata["livekit_agent_name"] as its dial identity, but nothing populated it: the value exists only in the agent process's rendered per-world environment, discarded after spawn. Carry it on the spawned-process handle and mirror it onto the world's runtime — exactly one distinct declared name sets the key; zero or conflicting names leave it absent (loudly, in the log) so the runner's typed pre-dial failure fires instead of dialing an arbitrarily chosen agent. Cold review: CLEARS — 4 mutations killed via the production path (one equivalent-mutant analyzed), 866 passed. Two contract notes ride for the next amendment: §3's metadata-keys wording, and a preflight rule for a static name under W>1. Signed-off-by: khushalsonawat --- src/fi/alk/harness/call_runner.py | 7 ++- src/fi/alk/harness/process_runtime.py | 31 ++++++++++ tests/harness/test_process_runtime.py | 83 ++++++++++++++++++++++++++- 3 files changed, 117 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py index 2b249da6..276e253b 100644 --- a/src/fi/alk/harness/call_runner.py +++ b/src/fi/alk/harness/call_runner.py @@ -180,9 +180,10 @@ def _check_config( def _dispatch_agent_name(runtime: EnvironmentRuntime) -> str | None: """The ONLY place this repo reads the dispatch-identity metadata key, so a - change to the key name/convention is a one-line adapt. `EnvironmentRuntime.metadata` defaults to `{}` and nothing - in `process_runtime.py` populates it yet -- every real "livekit" job - hits the caller's typed `CallAborted` below until a producer lands.""" + change to the key name/convention is a one-line adapt. The provisioner + mirrors the agent process's rendered LIVEKIT_AGENT_NAME here; a bundle + that declares none (or an ambiguous set) leaves the key absent and the + caller's typed `CallAborted` below fires.""" value = runtime.metadata.get("livekit_agent_name") return value.strip() if isinstance(value, str) and value.strip() else None diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py index 1272c142..af26ab11 100644 --- a/src/fi/alk/harness/process_runtime.py +++ b/src/fi/alk/harness/process_runtime.py @@ -936,6 +936,33 @@ class SpawnedWorldProcess: # it a second time. `None` in the local-lane fallback, same as `spawn_uid`/`spawn_gid` above. uid: int | None = None gid: int | None = None + # The voice dispatch identity (LIVEKIT_AGENT_NAME) exists only in the process's rendered + # per-world environment, which is discarded after spawn — carried here so the provider can + # surface it on the world's runtime metadata without re-rendering templates. + dispatch_agent_name: str | None = None + + +def _dispatch_metadata( + handles: "dict[str, SpawnedWorldProcess]", +) -> dict[str, "JsonValue"]: + """A world with exactly one voice agent gets its dispatch identity on the runtime metadata; + anything else leaves the key absent so the call runner's own typed pre-dial failure fires + instead of a call being dialed at an arbitrarily chosen agent. Ambiguity is loud here + because the pre-dial message cannot say WHY the key is missing.""" + names = sorted({ + name + for handle in handles.values() + if (name := (handle.dispatch_agent_name or "").strip()) + }) + if len(names) == 1: + return {"livekit_agent_name": names[0]} + if len(names) > 1: + logger.warning( + "world declares %d distinct LIVEKIT_AGENT_NAME values (%s); " + "leaving dispatch identity unset", + len(names), ", ".join(names), + ) + return {} # --- managed-engine launch commands --------------------------------------------------------- @@ -1276,6 +1303,7 @@ def spawn_source_process( process_name=process.name, handle=handle, port=port, world_index=world_index, uid=resolved_user.pw_uid if resolved_user is not None else None, gid=resolved_user.pw_gid if resolved_user is not None else None, + dispatch_agent_name=rendered.get("LIVEKIT_AGENT_NAME") or None, ) @@ -3739,16 +3767,19 @@ def _ensure_world(self, world_index: int) -> None: # live `EnvironmentRuntime` objects") reads as ONE object per world for the provider's # whole life. Minting a new object every rebuild meant `reset()`'s own state write (m5) # landed on an object no caller who captured an EARLIER reference would ever see again. + metadata = _dispatch_metadata(result.handles) if existing is not None: existing.runtime_id = new_runtime_id(self._bundle_digest, world_index) existing.endpoints = result.endpoints existing.state = RuntimeState.PREPARING + existing.metadata = metadata self._runtimes[world_index] = existing else: self._runtimes[world_index] = EnvironmentRuntime( runtime_id=new_runtime_id(self._bundle_digest, world_index), world_index=world_index, bundle_digest=self._bundle_digest, state=RuntimeState.PREPARING, endpoints=result.endpoints, + metadata=metadata, ) def _drop_world_shared_databases(self, world_index: int) -> None: diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py index eff5e836..ebee7f9a 100644 --- a/tests/harness/test_process_runtime.py +++ b/tests/harness/test_process_runtime.py @@ -29,7 +29,6 @@ from __future__ import annotations import asyncio -import importlib.util import json import subprocess import sys @@ -4578,3 +4577,85 @@ def test_provider_healthy_port_raises_typed_before_provision(tmp_path: Path) -> with pytest.raises(pr.ProcessRuntimeError) as excinfo: asyncio.run(provider.healthy(fake_runtime, work_directory=tmp_path)) assert excinfo.value.code == "internal_invariant_violated" + + +# --- voice dispatch identity on runtime metadata --------------------------------------------- + + +def test_spawn_source_process_carries_rendered_dispatch_agent_name(tmp_path: Path) -> None: + """The dial identity exists only in the rendered per-world env; the handle must carry the + RESOLVED value (world index substituted), and carry None when the process declares none.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + with_name = pr.spawn_source_process( + _source_process(environment={"LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}"}), + build_dir=build_dir, world_dir=tmp_path / "worlds" / "w2" / "svc", world_index=2, + port_plan=plan, configuration_addresses={}, secret_values={}, secret_purposes={}, + runner=fake_runner, + ) + assert with_name.dispatch_agent_name == "agent-w2" + + without = pr.spawn_source_process( + _source_process(environment={}), + build_dir=build_dir, world_dir=tmp_path / "worlds" / "w0" / "svc", world_index=0, + port_plan=plan, configuration_addresses={}, secret_values={}, secret_purposes={}, + runner=fake_runner, + ) + assert without.dispatch_agent_name is None + + +def test_dispatch_metadata_sets_the_key_only_for_exactly_one_distinct_name( + caplog: pytest.LogCaptureFixture, +) -> None: + """Zero agents -> absent (the call runner's typed pre-dial failure owns that outcome); + one distinct name (even from several handles) -> set; conflicting names -> absent, and + loudly, because the pre-dial message cannot say WHY the key is missing.""" + + def handle(name: str | None) -> pr.SpawnedWorldProcess: + return pr.SpawnedWorldProcess( + process_name="p", handle=FakeHandle(), port=1, world_index=0, + dispatch_agent_name=name, + ) + + assert pr._dispatch_metadata({}) == {} + assert pr._dispatch_metadata({"a": handle(None)}) == {} + assert pr._dispatch_metadata({"a": handle("agent-w0")}) == { + "livekit_agent_name": "agent-w0" + } + assert pr._dispatch_metadata({"a": handle("agent-w0"), "b": handle("agent-w0")}) == { + "livekit_agent_name": "agent-w0" + } + with caplog.at_level("WARNING"): + assert pr._dispatch_metadata( + {"a": handle("agent-w0"), "b": handle("other")} + ) == {} + assert any("distinct LIVEKIT_AGENT_NAME" in record.message for record in caplog.records) + + +def test_provision_surfaces_dispatch_identity_on_runtime_metadata(tmp_path: Path) -> None: + """End to end through provision(): a bundle whose agent process declares LIVEKIT_AGENT_NAME + lands the per-world RESOLVED value on runtime.metadata, which is exactly where the call + runner reads its dial identity.""" + + def add_dispatch_env(body: dict[str, Any]) -> dict[str, Any]: + for process in body["processes"]: + if process["name"] == "agent": + process["environment"]["LIVEKIT_AGENT_NAME"] = "agent-w{{WORLD_INDEX}}" + return body + + manifest = _manifest(add_dispatch_env) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + assert [runtime.metadata for runtime in runtimes] == [ + {"livekit_agent_name": "agent-w0"}, + {"livekit_agent_name": "agent-w1"}, + ] From 0b2e307dbcfba2ffe462bba2bb1203cf10dc4522 Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 27 Aug 2026 02:12:43 +0530 Subject: [PATCH 19/20] fix(harness): deliver mandatory terminal artifacts, redact stored receipts, support storeless worlds The platform refuses to acknowledge a complete artifact manifest that omits the build, result, and log kinds, so fully-completed runs were stamped failed. The guest now uploads those run-level artifacts before pushing the manifest, redacting receipt text and build output first so a stored artifact can never be the one copy that leaks a secret. An aborted run's manifest is marked complete, since nothing further will ever upload and the platform accepts an incomplete manifest only from a canceled attempt. A bundle with no SQL store is now runnable: the world factory yields a storeless world whose state accessors raise a typed error, which the scheduler reports as an errored receipt rather than a crash. Signed-off-by: khushalsonawat --- src/fi/alk/harness/hosted_entrypoint.py | 177 +++++++++++++++++++++-- src/fi/alk/harness/world/errors.py | 10 ++ tests/harness/test_hosted_entrypoint.py | 184 ++++++++++++++++++++++-- 3 files changed, 353 insertions(+), 18 deletions(-) diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 992629c0..3f93f919 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -64,6 +64,7 @@ RuntimeEndpoint, ) from .scenario_source import BundleScenarioSource, ScenarioDocumentInvalid, bundle_has_scenarios +from .world.errors import WorldStoreless from .world.handle import HostedWorld from .world.stores.postgres import AttachedPostgresStore @@ -312,8 +313,9 @@ async def build( class WorldFactoryError(RuntimeError): """The provisioner handed back a runtime this factory cannot build a `World` for — a bug - upstream (no postgres endpoint despite §2e's `no_sql_store` guarantee, or `build.json` missing - the row counts for that store), never a scenario-code fault.""" + upstream (no postgres endpoint despite `build.json` recording a seeded store, or `build.json` + missing the row counts for that store), never a scenario-code fault. A bundle that declared + no store at all is NOT this: that is the legal storeless lane (`StorelessWorld`, below).""" def _process_runtime_error_domain(exc: ProcessRuntimeError) -> FailureDomain: @@ -346,14 +348,11 @@ def _section_2f_code(code: str) -> str: return code if code in _SECTION_2F_CODES else "spawn_failed" -def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint: +def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint | None: for endpoint in runtime.endpoints.values(): if endpoint.protocol == "postgres": return endpoint - raise WorldFactoryError( - f"world {runtime.world_index}: no postgres-protocol endpoint in {sorted(runtime.endpoints)} " - "-- §2e's no_sql_store rule should make this unreachable" - ) + return None def load_build_output(work_directory: Path) -> dict[str, Any]: @@ -378,11 +377,57 @@ def row_counts_for_capability(build_output: dict[str, Any], capability: str) -> ) +class StorelessWorld: + """The `World` handle for a bundle that declared no SQL store — a stateless agent. + + Every state-touching method raises `WorldStoreless` (a `WorldError` subclass), which the + scheduler's phase classifier routes to the typed `world_usage` receipt failure: the scenario + errors by name, the run keeps going, and nothing crashes. A scenario whose setup/ready/checks + never touch state runs and passes exactly as it would anywhere else. `read_only()` returns the + handle itself — there is nothing writable to protect, and raising there would escape the + scheduler's phase machinery instead of landing in it.""" + + def __init__(self, world_index: int, rng: random.Random) -> None: + self.world_index = world_index + self.rng = rng + + def _refuse(self, operation: str) -> WorldStoreless: + return WorldStoreless( + f"{operation}: this bundle declares no SQL store, so its worlds hold no state — " + "a scenario check that needs state cannot run against a storeless bundle" + ) + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + raise self._refuse("state()") + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + raise self._refuse("put()") + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + raise self._refuse("change()") + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + raise self._refuse("drop()") + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Any: + raise self._refuse("call()") + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + raise self._refuse("query()") + + def read_only(self) -> "StorelessWorld": + return self + + class ProcessWorldFactory: """Builds a real `HostedWorld` over the runtime's postgres endpoint. `AttachedPostgresStore` (not the bare `PostgresStore`) is the correct base here — it takes a raw DSN and never manages a container's own lifecycle, matching a hosted world where `ProcessRuntimeProvider` already - owns the postgres process.""" + owns the postgres process. + + A runtime with no postgres endpoint forks on what `build.json` recorded: zero seeded stores + means the bundle is legitimately storeless (`StorelessWorld`); any recorded store means the + provisioner seeded one and then lost its endpoint — a bug upstream, still `WorldFactoryError`.""" def __init__(self, work_directory: Path) -> None: self._work_directory = work_directory @@ -390,6 +435,13 @@ def __init__(self, work_directory: Path) -> None: async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> World: endpoint = _find_postgres_endpoint(runtime) build_output = await asyncio.to_thread(load_build_output, self._work_directory) + if endpoint is None: + if build_output.get("stores"): + raise WorldFactoryError( + f"world {runtime.world_index}: build.json records seeded stores but the " + f"runtime has no postgres-protocol endpoint in {sorted(runtime.endpoints)}" + ) + return StorelessWorld(runtime.world_index, rng) row_counts = row_counts_for_capability(build_output, endpoint.capability) store = AttachedPostgresStore(endpoint.address) return await asyncio.to_thread( @@ -1109,6 +1161,97 @@ async def drain(self, *, complete: bool, deadline: float | None = None) -> bool: return self.is_fenced + +async def _upload_terminal_artifacts( + adapter: "OutboundAdapter", + work_directory: Path, + run_result: "RunResult | None", +) -> None: + """Channel 3's level table requires every complete manifest to carry `build`, + `result`, and `log` -- the platform refuses the manifest ack without exactly + those three, which stamped fully-completed runs as failed. At `traces` the + event trace is contract-required too, but the ack does not enforce it. + Best-effort throughout: a refusal or failure leaves the manifest listing only + what genuinely uploaded (the refusal log events raised this late are dropped + locally -- the run is already past its terminal).""" + deadline = adapter.deadline() + + build_path = work_directory / "artifacts" / "build.json" + try: + build_bytes = build_path.read_bytes() + except OSError: + build_bytes = json.dumps( + {"build_record": None, "reason": "build.json missing at finalize"} + ).encode("utf-8") + + receipts = list(run_result.receipts) if run_result is not None else [] + # Receipt text (reasons, failure fields) is redacted on every other outbound + # surface; the stored artifacts must not be the one copy that leaks a secret -- + # build.json can carry endpoint URLs with embedded credentials. + secret_values = adapter._extra_secret_values + try: + build_bytes = ob.redact_outbound_text( + build_bytes.decode("utf-8", errors="replace"), secret_values + ).encode("utf-8") + except Exception: + # Fail closed: a redaction that could not run must never let the raw + # bytes (which may carry credential-bearing endpoint URLs) reach storage. + build_bytes = json.dumps( + {"build_record": None, "reason": "build.json redaction failed at finalize"} + ).encode("utf-8") + await adapter.upload_artifact(build_bytes, kind=ob.ArtifactKind.BUILD, deadline=deadline) + + def _clean(value: str | None) -> str | None: + if value is None: + return None + return _cap_failure_message(ob.redact_outbound_text(value, secret_values)) + + result_doc = { + "schema_version": "futureagi.harness-result-set.v1", + "receipts": [ + { + "scenario_key": receipt.scenario_key, + "scenario_attempt": receipt.scenario_attempt, + "world_index": receipt.world_index, + "status": receipt.status, + "sub_goals": [ + {"name": goal.name, "held": goal.held, "judged": goal.judged, + "reason": _clean(goal.reason)} + for goal in receipt.sub_goals + ], + "failure": ( + {"code": _clean(receipt.failure.code), "stage": receipt.failure.stage, + "domain": receipt.failure.domain, + "message": _clean(receipt.failure.message)} + if receipt.failure is not None else None + ), + } + for receipt in receipts + ], + } + await adapter.upload_artifact( + json.dumps(result_doc, sort_keys=True).encode("utf-8"), + kind=ob.ArtifactKind.RESULT, deadline=deadline, + ) + + log_lines = [f"scenario {r.scenario_key} attempt {r.scenario_attempt}: {r.status}" for r in receipts] + log_lines.append(f"receipts: {len(receipts)}") + await adapter.upload_artifact( + ("\n".join(log_lines) + "\n").encode("utf-8"), + kind=ob.ArtifactKind.LOG, deadline=deadline, + ) + + spool_path = work_directory / EVENTS_SPOOL_DIR_NAME / "events.spool.jsonl" + try: + spool_bytes = spool_path.read_bytes() + except OSError: + spool_bytes = b"" + if spool_bytes: + await adapter.upload_artifact( + spool_bytes, kind=ob.ArtifactKind.TRACE, deadline=deadline, + ) + + def _evaluation_wire(evaluation: Any) -> dict[str, Any]: if evaluation.kind == "metric": return { @@ -1452,6 +1595,14 @@ async def _finish( if adapter.is_fenced: await _bounded_close() return EXIT_FENCED + if complete: + try: + await _upload_terminal_artifacts( + adapter, work_directory, + scheduler_result[1] if scheduler_result is not None else None, + ) + except Exception as exc: # noqa: BLE001 - post-terminal telemetry, never fatal + logger.error("terminal artifact upload failed: %s", exc) # the exit code comes from drain()'s own post-hoc fence check (deadline computed AFTER # emit_terminal, which is what arms the flush window), never a stale pre-drain read. fenced = await adapter.drain(deadline=adapter.deadline(), complete=complete) @@ -1749,10 +1900,15 @@ async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, m "domain": result.aborted.domain, "stage": HarnessStage.RUNNING.value, "code": result.aborted.code, "message": result.aborted.message, }, - complete=False, + # An aborted run's manifest is still COMPLETE -- nothing further + # will ever upload -- and the platform accepts an incomplete + # manifest only from a canceled attempt; sending false here made + # every aborted run unackable and let the gateway overwrite the + # real abort failure with terminal_delivery_incomplete. + complete=True, scheduler_result=(scheduler, result), ) - # `complete: true` only on a genuine, nothing-cut-short COMPLETED terminal. + # `complete: false` is reserved for the canceled lane alone. return await _finish(HarnessStage.COMPLETED, complete=True, scheduler_result=(scheduler, result)) finally: if pool is not None: @@ -1800,6 +1956,7 @@ def main(argv: list[str] | None = None) -> int: "NotWiredScenarioSource", "OutboundAdapter", "ProcessWorldFactory", + "StorelessWorld", "ScenarioPreallocationError", "ScenarioSource", "ScenarioSourceNotWired", diff --git a/src/fi/alk/harness/world/errors.py b/src/fi/alk/harness/world/errors.py index 5b1a9367..c8fafc47 100644 --- a/src/fi/alk/harness/world/errors.py +++ b/src/fi/alk/harness/world/errors.py @@ -66,3 +66,13 @@ class WorldUsageError(WorldError): saying which column `key` names — hosted worlds cannot invent tables or guess a column, so both are reported here rather than attempted. """ + + +class WorldStoreless(WorldError): + """The bundle declared no SQL store, so this world holds no state at all. + + A stateless agent's scenarios can still run and pass — but the moment scenario code asks the + handle for state (`state`, `put`, `query`, ...), there is nothing behind the handle to answer + with. Raised eagerly, by name, so the receipt says "this scenario's checks need a store the + bundle never declared" instead of crashing three layers down in a driver. + """ diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py index ba41230e..e330d624 100644 --- a/tests/harness/test_hosted_entrypoint.py +++ b/tests/harness/test_hosted_entrypoint.py @@ -140,6 +140,29 @@ def _write_bundle(root: Path) -> EnvironmentBundleV2: return EnvironmentBundleV2.model_validate(body) +def _write_storeless_bundle(root: Path) -> EnvironmentBundleV2: + """A legal stateless-agent bundle: no managed postgres process, zero capabilities, no seed. + What the relaxed store rule admits — the run below must work end-to-end against it.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + body["processes"] = [ + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": {"LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}"}, + "secret_purposes": ["target_provider"], "user": "svc-agent", "depends_on": [], + }, + ] + body["capabilities"] = {} + body["files"] = [] + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + def _job( *, connector: str = "vapi", parallelism: int = 1, artifacts: HarnessArtifactPolicy | None = None, @@ -1209,6 +1232,14 @@ async def scenario() -> None: assert payload["failure"]["stage"] == "running" assert payload["failure"]["code"] == "world_pool_exhausted" assert harness.provisioner.closed is True + # The aborted lane still finalizes like a completed one: nothing further will ever + # upload, so the manifest must say complete AND carry the three kinds the platform + # refuses to ack a complete manifest without. + assert harness.transport.manifests, "no artifact manifest was pushed" + final_manifest = harness.transport.manifests[-1] + assert final_manifest["complete"] is True + kinds = {entry["kind"] for entry in final_manifest["entries"]} + assert {"build", "result", "log"} <= kinds asyncio.run(scenario()) @@ -1260,11 +1291,18 @@ async def scenario() -> None: asyncio.run(scenario()) -def test_process_world_factory_raises_when_no_postgres_endpoint() -> None: - # `ProcessWorldFactory`/`_find_postgres_endpoint` had zero coverage -- only the pure - # `row_counts_for_capability` helper was unit-tested. +def test_process_world_factory_raises_when_stores_recorded_but_no_endpoint() -> None: + # The endpoint-lost guard: build.json records seeded stores but the runtime has no + # postgres endpoint -- a provisioner fault that must fail loudly, never silently degrade + # to a storeless world. build.json is present and readable so the guard, not the + # "build.json unreadable" path, is what fires. async def scenario() -> None: tmp = Path(tempfile.mkdtemp(prefix="p10-wf-endpoint-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": [{"capability": "database", "row_counts": {}}]}), + encoding="utf-8", + ) factory = he.ProcessWorldFactory(tmp) runtime = EnvironmentRuntime( runtime_id="digest:w0", world_index=0, bundle_digest="digest", @@ -1275,7 +1313,29 @@ async def scenario() -> None: except he.WorldFactoryError: pass else: - raise AssertionError("expected WorldFactoryError for a runtime with no postgres endpoint") + raise AssertionError( + "expected WorldFactoryError when build.json records stores but no endpoint exists" + ) + + asyncio.run(scenario()) + + +def test_process_world_factory_builds_storeless_world_when_no_endpoint_and_no_stores() -> None: + # The relaxed storeless path: no postgres endpoint AND build.json records no stores is a + # legal stateless bundle -- the factory hands back a StorelessWorld, not an error. + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-storeless-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": []}), encoding="utf-8" + ) + factory = he.ProcessWorldFactory(tmp) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + world = await factory.create(runtime, rng=random.Random(0)) + assert isinstance(world, he.StorelessWorld) asyncio.run(scenario()) @@ -1424,7 +1484,13 @@ async def scenario() -> None: assert len(harness.transport.manifests) >= 1 assert harness.transport.manifests[-1]["complete"] is True - assert len(harness.transport.manifests[-1]["entries"]) == 2 + # Channel 3's level table: a complete manifest carries the run-level kinds + # (`build`, `result`, `log`, plus the event trace) alongside the transcripts -- + # the platform refuses the manifest ack without them. + manifest_kinds = [e["kind"] for e in harness.transport.manifests[-1]["entries"]] + assert manifest_kinds.count("transcript") == 2 + for required in ("build", "result", "log"): + assert required in manifest_kinds, manifest_kinds assert harness.provisioner.provision_calls >= 1 assert harness.provisioner.closed is True @@ -1788,7 +1854,15 @@ async def scenario() -> None: ) code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) assert code == he.EXIT_OK - assert harness.transport.artifacts == {} # zero bytes reached the transport + # metadata-only forbids transcripts, but the level table MANDATES the run-level + # kinds -- so exactly build/result/log reach the transport, and nothing scenario-scoped. + entries = harness.transport.manifests[-1]["entries"] + uploaded_kinds = sorted(entry["kind"] for entry in entries) + assert "transcript" not in uploaded_kinds + for required in ("build", "result", "log"): + assert required in uploaded_kinds, uploaded_kinds + listed_digests = {e["artifact_id"].split(":", 1)[1] for e in entries} + assert set(harness.transport.artifacts) == listed_digests # no unlisted bytes receipt_body = harness.transport.receipts[("job-1", "s1")] assert receipt_body["status"] == "passed" # a refused upload must not error the scenario @@ -1852,7 +1926,12 @@ async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> Call assert receipt_body["call"]["recording_artifacts"] == [] # recordings refused at traces transcript_digest = receipt_body["call"]["transcript_artifact"].split(":", 1)[1] - assert set(harness.transport.artifacts) == {transcript_digest} # the recording never uploaded + assert transcript_digest in harness.transport.artifacts + uploaded_kinds = {e["kind"] for e in harness.transport.manifests[-1]["entries"]} + assert "recording_combined" not in uploaded_kinds # the recording never uploaded + assert {"build", "result", "log"} <= uploaded_kinds + recording_digest = hashlib.sha256(b"recording-bytes").hexdigest() + assert recording_digest not in harness.transport.artifacts # not even the bytes log_events = [r for r in harness.transport.event_records if r.get("type") == "log"] assert any( @@ -2035,6 +2114,37 @@ async def scenario() -> None: asyncio.run(scenario()) +def test_stored_result_artifact_redacts_a_secret_spanning_the_cap_boundary() -> None: + # `_upload_terminal_artifacts._clean` must redact BEFORE capping -- capping first can cut a + # secret in half at the 4KB boundary, and exact-substring redaction can no longer find the + # surviving fragment, making the stored `result` artifact the one outbound copy that leaks. + async def scenario() -> None: + secret = "sk-" + "S" * 33 # 36 chars, spanning the 4084-char truncation point + transport = FakeTransport() + adapter = _build_adapter(transport, extra_secret_values=(secret,)) + work = Path(tempfile.mkdtemp(prefix="p10-redact-cap-")) + receipt = ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", + message="x" * 4070 + secret, + ), + ) + await he._upload_terminal_artifacts( + adapter, work, RunResult(receipts=(receipt,), aborted=None) + ) + result_blobs = [b for b in transport.artifacts.values() if b"harness-result-set" in b] + assert result_blobs + blob = result_blobs[0] + fragments = [ + secret[i : i + 8] for i in range(len(secret) - 7) if secret[i : i + 8].encode() in blob + ] + assert not fragments, f"secret fragment survived into the stored result artifact: {fragments[:3]}" + + asyncio.run(scenario()) + + def test_cancel_before_provision_skips_provisioning_entirely() -> None: # The pre-provision cancel checkpoint (`hosted_entrypoint.py:1342` area) had no # test driving a cancel signal written BEFORE `run_job` starts -- disabling all three @@ -2723,6 +2833,63 @@ async def scenario() -> None: # ================================================================================================= +def test_storeless_bundle_run_with_state_free_checks_passes_end_to_end() -> None: + # The relaxed store rule end-to-end: a bundle with zero postgres capabilities clears + # preflight, the real `ProcessWorldFactory` hands the scheduler a `StorelessWorld` (the fake + # provisioner's runtimes carry no endpoints and build.json records no stores), and a scenario + # whose checks never touch state passes exactly as it would against a postgres world. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, bundle_writer=_write_storeless_bundle, + build_output={"stores": []}, + ) + harness.deps.build_world_factory = he.ProcessWorldFactory + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + receipt = harness.transport.receipts[("job-1", "s1")] + assert receipt["status"] == "passed" + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + +def test_storeless_bundle_check_touching_state_fails_typed_never_crashes() -> None: + # A check calling `world.state()` against a storeless world must land in the scheduler's + # typed `world_usage` lane — an errored receipt naming the missing store — while the run + # itself still completes cleanly. + @dataclass + class StateTouchingSubGoal: + name: str + judged: str = "yes" + + def check(self, world: Any, calls: Any) -> object: + del calls + world.state() + return None + + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [StateTouchingSubGoal("reads-state")])] + harness = _build_harness( + scenarios=scenarios, bundle_writer=_write_storeless_bundle, + build_output={"stores": []}, + ) + harness.deps.build_world_factory = he.ProcessWorldFactory + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + receipt = harness.transport.receipts[("job-1", "s1")] + assert receipt["status"] == "errored" + assert receipt["failure"]["code"] == "world_usage" + assert "no SQL store" in receipt["failure"]["message"] + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + def test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure() -> None: # Before the fix: `sys.exit(0)` at module level inside `setup.py` propagated as a raw # `SystemExit` straight out of `run_job` -- the guest process itself would exit 0 with ZERO @@ -2842,7 +3009,8 @@ async def scenario() -> None: test_fence_landing_on_the_final_drain_still_exits_fenced, test_fence_from_scenarios_client_exits_fenced_not_crashed, test_scenarios_channel_uses_bearer_auth_never_api_key, - test_process_world_factory_raises_when_no_postgres_endpoint, + test_process_world_factory_raises_when_stores_recorded_but_no_endpoint, + test_process_world_factory_builds_storeless_world_when_no_endpoint_and_no_stores, test_process_world_factory_raises_when_build_json_has_no_matching_store, test_build_json_two_stores_emit_two_baseline_frozen_events, test_build_json_degrade_payload_matches_the_recorded_values, From 7c16cbdd2a21249dec113f862ec6437512484f9d Mon Sep 17 00:00:00 2001 From: khushalsonawat Date: Thu, 27 Aug 2026 02:12:51 +0530 Subject: [PATCH 20/20] feat(harness): preflight guards for per-world agent names and storeless bundles At parallelism greater than one, a static LIVEKIT_AGENT_NAME registers every world's agent under the same identity, so a dispatch lands on an arbitrary world and evidence silently crosses worlds; preflight now requires the per-world placeholder. A stateless bundle (no SQL store) is accepted, but one declaring the tool_trace evidence seam is rejected, since that seam reads its evidence from the world's postgres store. Signed-off-by: khushalsonawat --- src/fi/alk/harness/process_preflight.py | 35 ++++++++++++-- tests/harness/test_process_preflight.py | 61 +++++++++++++++++++++---- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py index 1fd16078..d104f0ad 100644 --- a/src/fi/alk/harness/process_preflight.py +++ b/src/fi/alk/harness/process_preflight.py @@ -7,7 +7,7 @@ `unknown_field` translation, and every rule that needs the job the bundle will run under (placeholder vocabulary, secret purposes against the job's `secret_refs`, the `depends_on` graph, the engine catalog, `seed_missing`, `inputs_digest` verification, reserved-name content scanning, -`no_sql_store`, and resource sanity). `seed_strategy_unsupported`, `sentinel_shape_mismatch`, +store/evidence-seam coherence, and resource sanity). `seed_strategy_unsupported`, `sentinel_shape_mismatch`, `capability_unresolved`, `configuration_name_duplicate`, `user_assignment_invalid`, and `capability_engine_mismatch` are already enforced by the model layer and are not repeated here. @@ -37,6 +37,7 @@ BUNDLE_V2_SCHEMA_VERSION, BundleFileV2, EnvironmentBundleV2, + EvidenceSeam, ManagedEngine, ManagedProcess, RuntimeKindV2, @@ -204,7 +205,7 @@ def preflight_bundle( _verify_reserved_names(bundle_dir, manifest) # 5 _verify_seed_files_on_disk_and_listed(bundle_dir, manifest, files) # 5 - _verify_no_sql_store(manifest) # 6 + _verify_no_sql_store(manifest) # 6 (relaxed: storeless bundles are legal) _verify_resource_sanity(manifest, parallelism=parallelism) # 7 @@ -614,18 +615,28 @@ def _verify_seed_files_on_disk_and_listed( ) -# --- item 6: no_sql_store ------------------------------------------------------------------------ +# --- item 6: store / evidence-seam coherence ----------------------------------------------------- def _verify_no_sql_store(manifest: EnvironmentBundleV2) -> None: + """A `kind: process` bundle with zero postgres-protocol capabilities is a legal stateless + agent: its worlds hold no state, and only scenario code that actually asks for state gets a + typed failure (`ProcessWorldFactory`'s storeless handle). The one thing a storeless bundle can + never satisfy is `evidence_seam: tool_trace` — that seam reads the agent's tool calls out of + the world's own postgres database, so accepting it here would guarantee zero evidence at + runtime and fail every scenario with `evidence_missing` instead of a preflight verdict.""" if manifest.runtime.kind is not RuntimeKindV2.PROCESS: return - if not any( + if any( capability.protocol is CapabilityProtocol.POSTGRES for capability in manifest.capabilities.values() ): + return + if manifest.runtime.evidence_seam is EvidenceSeam.TOOL_TRACE: raise PreflightError( - "no_sql_store", "kind: process requires at least one postgres-protocol capability" + "evidence_seam_unsatisfiable", + "evidence_seam: tool_trace reads evidence from a postgres store, and this bundle " + "declares no postgres-protocol capability", ) @@ -643,6 +654,20 @@ def _verify_resource_sanity(manifest: EnvironmentBundleV2, *, parallelism: int) "parallelism_out_of_range", f"parallelism={parallelism} is outside {_MIN_PARALLELISM}..{_MAX_PARALLELISM}", ) + if parallelism > 1: + # Concurrent worlds dial their agents by LiveKit identity. A static + # agent name registers every world's agent under the same identity, so + # a dispatch lands on an arbitrary world — evidence silently crosses + # worlds. The {{WORLD_INDEX}} placeholder is what makes names per-world. + for process in manifest.processes: + environment = getattr(process, "environment", None) or {} + value = environment.get("LIVEKIT_AGENT_NAME") + if isinstance(value, str) and value and "{{WORLD_INDEX}}" not in value: + raise PreflightError( + "agent_name_not_world_unique", + f"process {process.name} renders a static LIVEKIT_AGENT_NAME " + f"with parallelism={parallelism}; include {{{{WORLD_INDEX}}}}", + ) __all__ = ["PreflightError", "preflight_bundle"] diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py index 2000e9cf..1b1cc187 100644 --- a/tests/harness/test_process_preflight.py +++ b/tests/harness/test_process_preflight.py @@ -646,20 +646,37 @@ def mutate(body: dict[str, Any]) -> dict[str, Any]: preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) -# --- item 6: no_sql_store ------------------------------------------------------------------------ +# --- item 6: store / evidence-seam coherence ----------------------------------------------------- -def test_a_process_bundle_with_no_postgres_capability_is_rejected(tmp_path: Path) -> None: - """Keeps the `database` capability (so the `{{DATABASE_URL}}` placeholder in `agent`'s - environment still resolves) and only changes its protocol away from postgres, isolating this - from the placeholder-vocabulary check that would otherwise fire first.""" +def _storeless(body: dict[str, Any]) -> dict[str, Any]: + """Zero postgres-protocol capabilities. Keeps the `database` capability slug (so the + `{{DATABASE_URL}}` placeholder in `agent`'s environment still resolves) and only changes its + protocol away from postgres, isolating these tests from the placeholder-vocabulary check that + would otherwise fire first.""" + body["capabilities"]["database"]["protocol"] = "http" + body["seed"] = None + return body + + +def test_a_storeless_process_bundle_passes_preflight(tmp_path: Path) -> None: + # A stateless agent is legal: nothing else in the checklist needs a store, and the runtime's + # storeless world only fails, by type, when scenario code actually asks it for state. + manifest = _build_bundle(tmp_path, body_overrides=_storeless, include_seed=False) + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_storeless_bundle_declaring_tool_trace_evidence_is_rejected(tmp_path: Path) -> None: + # `tool_trace` reads the agent's tool calls out of the world's postgres store — with no store + # the seam can never produce evidence, so it must fail here, not as `evidence_missing` on + # every scenario at runtime. def mutate(body: dict[str, Any]) -> dict[str, Any]: - body["capabilities"]["database"]["protocol"] = "http" - body["seed"] = None + body = _storeless(body) + body["runtime"]["evidence_seam"] = "tool_trace" return body manifest = _build_bundle(tmp_path, body_overrides=mutate, include_seed=False) - with pytest.raises(PreflightError, match="no_sql_store"): + with pytest.raises(PreflightError, match="evidence_seam_unsatisfiable"): preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) @@ -912,6 +929,10 @@ def _raised_codes(source_text: str, callee_name: str, *, whole_first_argument: b "fixed_port_reserved", # §2e, v1.9. }) _SECTION_2E_MECHANICAL_CODES = frozenset({ + "agent_name_not_world_unique", + # storeless bundles: a bundle with no postgres capability cannot satisfy `evidence_seam: + # tool_trace` — the §2e contract table still needs the amendment adding this code. + "evidence_seam_unsatisfiable", "bundle_schema_unsupported", "bundle_manifest_invalid", "bundle_manifest_drifted", "bundle_digest_mismatch", "bundle_digest_invalid", "inputs_digest_invalid", "file_sha256_invalid", "source_digest_invalid", "bundle_file_missing", @@ -1056,3 +1077,27 @@ def test_the_section_2f_extraction_itself_finds_a_nonempty_set() -> None: assert "spawn_failed" in raised assert "source_tree_unavailable" in raised assert "unsupported_capability_protocol" in raised + +def test_parallel_static_agent_name_is_refused(tmp_path: Path) -> None: + def overrides(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["LIVEKIT_AGENT_NAME"] = "static-agent" + return body + + manifest = _build_bundle(tmp_path, body_overrides=overrides) + with pytest.raises(PreflightError, match="agent_name_not_world_unique"): + preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + + +def test_parallel_world_indexed_agent_name_passes_the_guard(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + + +def test_single_world_static_agent_name_is_allowed(tmp_path: Path) -> None: + def overrides(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["LIVEKIT_AGENT_NAME"] = "static-agent" + return body + + manifest = _build_bundle(tmp_path, body_overrides=overrides) + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) +