diff --git a/src/fi/alk/harness/bundle_v2.py b/src/fi/alk/harness/bundle_v2.py new file mode 100644 index 00000000..9cee66ee --- /dev/null +++ b/src/fi/alk/harness/bundle_v2.py @@ -0,0 +1,695 @@ +"""`futureagi.environment-bundle.v2` — the hosted provisioner's manifest shape (`hosted-execution- +seams.md` v1.9). + +v1 (`bundle.py`) describes a `command`-per-service compose world and embeds the repository +source. v2 describes `/work/source` as already present and a job that starts plain processes on +localhost: `processes` (managed engines and copied-and-built source trees), `seed` (how each +store's baseline is built and proven), and the same `capabilities`/`readiness`/`files`/ +`provenance` shape widened for both. v1 stays untouched — this module is additive, not a +replacement, and the two schema versions are never interchangeable: a hosted provisioner that +receives a `…bundle.v1` manifest rejects it rather than guessing. + +What lives here is model-layer only: the shapes, the closed vocabularies, and the rules that need +nothing but the manifest's own fields to decide. Rules that need the bundle's actual files (secret +scanning, digest/file verification) or the job it will run under (`compose_not_hosted`, +`engine_unsupported`, `no_sql_store`, the `depends_on` graph, placeholder-vocabulary checking, +reserved-name scanning of migration content) are the §2e preflight checklist's job, not this +module's — see `hosted-execution-seams.md` §2e. Also deferred to that preflight: translating +pydantic's `extra="forbid"` rejection of an unknown process-entry key into §2b's `unknown_field` +code — that translation belongs where error surfacing is owned, not here. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter +from enum import Enum +from pathlib import Path +from typing import Annotated, Any, Literal, Sequence, Union + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + JsonValue, + ValidationError, + field_validator, + model_validator, +) + +from .bundle import CapabilityProtocol, _reject_secret_values, _safe_relative + +BUNDLE_V2_SCHEMA_VERSION = "futureagi.environment-bundle.v2" +BUNDLE_V2_MANIFEST = "manifest.json" + + +class BundleV2Error(RuntimeError): + """A v2 bundle manifest is wrong-versioned, malformed, or fails a model-layer rule.""" + + +# --- §2a runtime ------------------------------------------------------------------------------- + + +class RuntimeKindV2(str, Enum): + PROCESS = "process" + EXTERNAL = "external" + COMPOSE = "compose" + + +class EvidenceSeam(str, Enum): + HTTP_TOOL = "http_tool" + TOOL_TRACE = "tool_trace" + + +class BundleRuntimeV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + kind: RuntimeKindV2 + control_service: str | None = None + evidence_seam: EvidenceSeam | None = None + # Carried over from v1 for `kind: compose` only (local SDK runs); a hosted `process` bundle + # has no document to point at, since `/work/source` is already on disk. + document: str | None = None + + @model_validator(mode="after") + def _kind_specific_rules(self) -> "BundleRuntimeV2": + if self.kind is RuntimeKindV2.PROCESS and self.evidence_seam is None: + raise ValueError("evidence_seam_required: kind=process") + if self.kind is RuntimeKindV2.COMPOSE and not self.document: + raise ValueError("compose_runtime_requires_document") + if self.kind is not RuntimeKindV2.COMPOSE and self.document is not None: + raise ValueError("document_only_for_compose") + if self.document: + _safe_relative(self.document) + return self + + +# --- §2b processes ------------------------------------------------------------------------- + + +class ProcessKind(str, Enum): + MANAGED = "managed" + SOURCE = "source" + + +class ManagedEngine(str, Enum): + POSTGRES = "postgres" + REDIS = "redis" + RABBITMQ = "rabbitmq" + + +class ProcessUser(str, Enum): + """The snapshot's fixed, bundle-assignable users (§0). `svc-control` runs ALK itself and is + never a process's own user.""" + + SVC_AGENT = "svc-agent" + SVC_TOOLS = "svc-tools" + SVC_DATA = "svc-data" + + +class SecretPurpose(str, Enum): + TARGET_PROVIDER = "target_provider" + SOURCE_CHECKOUT = "source_checkout" + + +# §0 (v1.8): a process `name` is path-joined into `/work/build//` and +# `/work/worlds/w//` verbatim (§2b) — the pattern below is the closed shape that makes +# `/`, `..`, and an absolute form unspellable at the model layer, matching every §2b example +# (including `tools-api`). +_PROCESS_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]*$") + + +def _validate_process_name(name: str) -> str: + if not _PROCESS_NAME_PATTERN.fullmatch(name): + raise ValueError(f"process_name_invalid: {name!r} must match ^[a-z0-9][a-z0-9_-]*$") + return name + + +class StartedCheck(BaseModel): + model_config = ConfigDict(extra="forbid") + + # §2b (v1.8): "the value selects the port-probe variant, it is not a literal port number" — + # the probed port is always the dependency's own allocated port (`port_plan.port_for`, + # `process_runtime.py`), honoring `fixed_port` when the process declares one. A prior version + # of this field carried a literal int; `bool` makes the "not a literal" rule unspellable + # wrong rather than merely documented. + port: bool | None = None + log_marker: str | None = None + timeout_seconds: float = Field(default=30.0, gt=0) + + @model_validator(mode="after") + def _exactly_one_probe(self) -> "StartedCheck": + has_port = bool(self.port) + has_marker = self.log_marker is not None + if has_port == has_marker: + raise ValueError("started_check_requires_exactly_one_of_port_or_log_marker") + return self + + +class ManagedProcess(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal[ProcessKind.MANAGED] = ProcessKind.MANAGED + engine: ManagedEngine + version: str = Field(min_length=1) + user: ProcessUser + depends_on: list[str] = Field(default_factory=list) + + @field_validator("name") + @classmethod + def _name_shape(cls, value: str) -> str: + return _validate_process_name(value) + + +class SourceProcess(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal[ProcessKind.SOURCE] = ProcessKind.SOURCE + working_directory: str + build_commands: list[list[str]] = Field(default_factory=list) + run_command: list[str] = Field(min_length=1) + environment: dict[str, str] = Field(default_factory=dict) + build_environment: dict[str, str] | None = None + fixed_port: int | None = Field(default=None, ge=1, le=65535) + started_check: StartedCheck | None = None + secret_purposes: list[SecretPurpose] = Field(default_factory=list) + user: ProcessUser + depends_on: list[str] = Field(default_factory=list) + + @field_validator("name") + @classmethod + def _name_shape(cls, value: str) -> str: + return _validate_process_name(value) + + @model_validator(mode="after") + def _shape(self) -> "SourceProcess": + _safe_relative(self.working_directory) + for step in self.build_commands: + if not step: + raise ValueError("build_command_step_empty") + return self + + +ProcessEntry = Annotated[Union[ManagedProcess, SourceProcess], Field(discriminator="kind")] + + +# --- §2c seed ------------------------------------------------------------------------------ + + +class BaselineStrategy(str, Enum): + TEMPLATE_DATABASE = "template_database" + DATADIR_COPY = "datadir_copy" + EMPTY = "empty" + + +# The §2b catalog table. The engine a store answers to is the *capability's* protocol (resolved in +# the root validator, where `capabilities` is in scope) — a store entry carries no `engine` field +# of its own, and the sentinel's shape is a proof of that engine, not a second source for it. +_ENGINE_STRATEGIES: dict[ManagedEngine, frozenset[BaselineStrategy]] = { + ManagedEngine.POSTGRES: frozenset( + {BaselineStrategy.TEMPLATE_DATABASE, BaselineStrategy.DATADIR_COPY} + ), + ManagedEngine.REDIS: frozenset({BaselineStrategy.DATADIR_COPY, BaselineStrategy.EMPTY}), + ManagedEngine.RABBITMQ: frozenset({BaselineStrategy.DATADIR_COPY}), +} + + +class StoreBaseline(BaseModel): + model_config = ConfigDict(extra="forbid") + + strategy: BaselineStrategy + inputs_digest: str + + @model_validator(mode="after") + def _digest_shape(self) -> "StoreBaseline": + if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.inputs_digest): + raise ValueError("inputs_digest_invalid") + return self + + +class Sentinel(BaseModel): + """A store's per-protocol read-only proof, per §2c: postgres `{query, expected}`, redis + `{key, expected}`, rabbitmq `{queue, expected_depth}` — exactly one shape, never a mix.""" + + model_config = ConfigDict(extra="forbid") + + query: str | None = None + expected: str | None = None + key: str | None = None + queue: str | None = None + expected_depth: int | None = Field(default=None, ge=0) + + @model_validator(mode="after") + def _one_protocol_shape(self) -> "Sentinel": + if self.implied_engine is None: + raise ValueError( + "sentinel_shape_invalid: expected exactly one of " + "postgres{query,expected}, redis{key,expected}, rabbitmq{queue,expected_depth}" + ) + return self + + @property + def implied_engine(self) -> ManagedEngine | None: + postgres = self.query is not None and self.expected is not None + redis = self.key is not None and self.expected is not None + rabbitmq = self.queue is not None and self.expected_depth is not None + shapes = [ + (postgres, ManagedEngine.POSTGRES, {self.key, self.queue, self.expected_depth}), + (redis, ManagedEngine.REDIS, {self.query, self.queue, self.expected_depth}), + (rabbitmq, ManagedEngine.RABBITMQ, {self.query, self.key, self.expected}), + ] + matched = [ + engine for present, engine, others in shapes if present and others == {None} + ] + return matched[0] if len(matched) == 1 else None + + +class StoreEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + + capability: str = Field(min_length=1) + migrations: list[str] = Field(default_factory=list) + seed_files: list[str] = Field(default_factory=list) + baseline: StoreBaseline + sentinel: Sentinel + + @model_validator(mode="after") + def _paths(self) -> "StoreEntry": + for relative_path in (*self.migrations, *self.seed_files): + _safe_relative(relative_path) + return self + + +class Seed(BaseModel): + model_config = ConfigDict(extra="forbid") + + stores: list[StoreEntry] = Field(default_factory=list) + + +# --- §2d capabilities, readiness, files, provenance ----------------------------------------- + + +# §2b's closed placeholder vocabulary, mirrored here (not imported from `process_preflight.py`, +# which imports this module) so a `configuration_name` can never shadow a builtin token — the +# reverse dependency direction is preflight -> model, not model -> preflight. +_RESERVED_CONFIGURATION_NAMES = {"WORLD_INDEX", "WORLD_DIR", "DB_NAME"} +_RESERVED_CONFIGURATION_PREFIX = re.compile(r"^(PORT|HOST)_") + + +class CapabilityV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + protocol: CapabilityProtocol + service: str = Field(min_length=1) + container_port: int | None = Field(default=None, ge=1, le=65535) + configuration_name: str | None = None + + @model_validator(mode="after") + def _configuration_name_not_reserved(self) -> "CapabilityV2": + # A `configuration_name` colliding with a fixed placeholder or a `{{PORT_/HOST_}}` prefix + # would render the builtin token instead of this capability's address, with no error and + # no way for the producer to spell the intended value (F8, p4-round1-review). + name = self.configuration_name + if name and ( + name in _RESERVED_CONFIGURATION_NAMES or _RESERVED_CONFIGURATION_PREFIX.match(name) + ): + raise ValueError(f"configuration_name_reserved: {name}") + return self + + +class ReadinessProbeV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + capability: str + path: str | None = None + timeout_seconds: float = Field(default=120.0, gt=0, le=1800) + interval_seconds: float = Field(default=1.0, gt=0, le=60) + + +class BundleFileV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + path: str + sha256: str + size: int = Field(ge=0) + + @model_validator(mode="after") + def _valid_path(self) -> "BundleFileV2": + _safe_relative(self.path) + if not re.fullmatch(r"[0-9a-f]{64}", self.sha256): + raise ValueError(f"file_sha256_invalid: {self.path}") + return self + + +class BundleProvenanceV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + source_kind: str + repository: str | None = None + commit: str | None = None + source_digest: str + generator: str = "fi.alk.harness" + generator_version: str = "1" + adopted_files: list[str] = Field(default_factory=list) + generated_files: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _valid_source_digest(self) -> "BundleProvenanceV2": + # Bare 64-hex, matching what `source_fingerprint` (v1's producer) actually emits — no + # `sha256:` prefix, unlike `digest`/`inputs_digest`. + if not re.fullmatch(r"[0-9a-f]{64}", self.source_digest): + raise ValueError(f"source_digest_invalid: {self.source_digest}") + return self + + +# --- manifest root --------------------------------------------------------------------------- + +_CAPABILITY_SLUG = re.compile(r"[a-z][a-z0-9_]*") + +# §2c: a store's engine is the engine behind its capability's protocol, not a field the store +# carries itself. Only postgres/redis/amqp capabilities can host a store at all (§2c); any other +# protocol on a store's capability is a producer error §2e is left to catch. +_STORE_ENGINE_BY_PROTOCOL: dict[CapabilityProtocol, ManagedEngine] = { + CapabilityProtocol.POSTGRES: ManagedEngine.POSTGRES, + CapabilityProtocol.REDIS: ManagedEngine.REDIS, + CapabilityProtocol.AMQP: ManagedEngine.RABBITMQ, +} + + +class EnvironmentBundleV2(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str + digest: str + name: str + runtime: BundleRuntimeV2 + processes: list[ProcessEntry] = Field(default_factory=list) + seed: Seed | None = None + capabilities: dict[str, CapabilityV2] = Field(default_factory=dict) + readiness: list[ReadinessProbeV2] = Field(default_factory=list) + files: list[BundleFileV2] = Field(default_factory=list) + provenance: BundleProvenanceV2 + metadata: dict[str, JsonValue] = Field(default_factory=dict) + + @model_validator(mode="after") + def _validate_manifest(self) -> "EnvironmentBundleV2": + if self.schema_version != BUNDLE_V2_SCHEMA_VERSION: + raise ValueError(f"bundle_schema_unsupported: {self.schema_version}") + if not re.fullmatch(r"sha256:[0-9a-f]{64}", self.digest): + raise ValueError("bundle_digest_invalid") + + if self.runtime.kind is RuntimeKindV2.PROCESS and not self.processes: + raise ValueError("processes_required: kind=process") + if self.runtime.kind is RuntimeKindV2.EXTERNAL and ( + self.processes or self.seed is not None + ): + raise ValueError("processes_and_seed_forbidden: kind=external") + + for slug in self.capabilities: + if not _CAPABILITY_SLUG.fullmatch(slug): + raise ValueError(f"capability_slug_invalid: {slug}") + + process_names = Counter(process.name for process in self.processes) + duplicated_names = sorted(name for name, count in process_names.items() if count > 1) + if duplicated_names: + raise ValueError("process_name_duplicate: " + ", ".join(duplicated_names)) + known_names = set(process_names) + processes_by_name = {process.name: process for process in self.processes} + + # B3 (p3-round2-review): only `kind: process` has a `processes` array to resolve against — + # `external` omits `processes` entirely (§2a) and `compose` addresses services through its + # own `document`, not this array. Gating here, rather than by emptying `known_names`, + # keeps the duplicate-name check above meaningful for every runtime kind. + if self.runtime.kind is RuntimeKindV2.PROCESS: + service_unresolved = { + slug: capability.service + for slug, capability in self.capabilities.items() + if capability.service not in known_names + } + if service_unresolved: + detail = ", ".join( + f"{slug}: {service}" for slug, service in sorted(service_unresolved.items()) + ) + raise ValueError(f"service_unresolved: {detail}") + + control_service = self.runtime.control_service + if control_service is not None and control_service not in known_names: + raise ValueError(f"control_service_unresolved: {control_service}") + if control_service is not None and isinstance( + processes_by_name[control_service], ManagedProcess + ): + # §2a: control_service is the agent-side service the world handle and evidence + # seam attach to — a datastore in that role is incoherent, and would otherwise + # silently resolve and take svc-agent below (N9, p4-round2-review). + raise ValueError( + f"control_service_unresolved: {control_service} is a managed engine, not a " + "source process" + ) + + # §2b/§0 (v1.6): the snapshot's SERVICE users are assigned by role, not authored — + # the control service gets svc-agent, every other source process svc-tools, every + # managed engine svc-data. Decidable from the manifest's own fields alone once + # `control_service` is resolved, which is why it lands here rather than in preflight + # (F5, p4-round1-review). + for process in self.processes: + if isinstance(process, ManagedProcess): + expected_user = ProcessUser.SVC_DATA + elif process.name == control_service: + expected_user = ProcessUser.SVC_AGENT + else: + expected_user = ProcessUser.SVC_TOOLS + if process.user is not expected_user: + raise ValueError( + f"user_assignment_invalid: {process.name} must be " + f"{expected_user.value}, got {process.user.value}" + ) + + names_to_slugs: dict[str, list[str]] = {} + for slug, capability in self.capabilities.items(): + if capability.configuration_name: + names_to_slugs.setdefault(capability.configuration_name, []).append(slug) + duplicated = {name: slugs for name, slugs in names_to_slugs.items() if len(slugs) > 1} + if duplicated: + detail = ", ".join( + f"{name} ({', '.join(sorted(slugs))})" for name, slugs in sorted(duplicated.items()) + ) + raise ValueError(f"configuration_name_duplicate: {detail}") + + unresolved = { + probe.capability + for probe in self.readiness + if probe.capability not in self.capabilities + } + if self.seed is not None: + for store in self.seed.stores: + if store.capability not in self.capabilities: + unresolved.add(store.capability) + if unresolved: + raise ValueError("capability_unresolved: " + ", ".join(sorted(unresolved))) + + # B1 (p3-round2-review): a capability's *declared* protocol can disagree with the process + # actually backing it. F19 (p4-round1-review) widened this from "only capabilities with a + # seed store" to every capability whose protocol names a managed engine — a redis + # capability with no store entry at all (used only for a `{{...}}` address, never seeded) + # was previously never checked, and could point `service` at a postgres process silently. + for slug, capability in self.capabilities.items(): + engine = _STORE_ENGINE_BY_PROTOCOL.get(capability.protocol) + if engine is None: + continue + backing = processes_by_name.get(capability.service) + if isinstance(backing, ManagedProcess) and backing.engine is not engine: + raise ValueError( + f"capability_engine_mismatch: {slug}: protocol {capability.protocol.value} " + f"resolves to {engine.value}, but {capability.service} is a " + f"{backing.engine.value} process" + ) + + if self.seed is not None: + # Every store's capability resolved above, so its protocol is known — that protocol, + # not the sentinel's own shape, is the authoritative engine (§2c classifies stores by + # capability protocol; the sentinel only proves that engine, it doesn't select it). + for store in self.seed.stores: + capability = self.capabilities[store.capability] + engine = _STORE_ENGINE_BY_PROTOCOL.get(capability.protocol) + if engine is None: + # B2 (p3-round2-review): a store on a capability outside the three protocols + # this module knows how to seed (http, mongodb, ...) has no engine to check + # its sentinel/strategy against — a producer error, not a silent pass-through. + raise ValueError( + f"store_protocol_unsupported: {store.capability}: protocol " + f"{capability.protocol.value} cannot host a seed store" + ) + backing = processes_by_name.get(capability.service) + if not isinstance(backing, ManagedProcess): + # F19 (p4-round1-review): a store on a capability backed by a `SourceProcess` + # has no managed engine to migrate or seed at all — the all-capabilities + # engine pass above only fires for a *wrong* managed engine, not a missing one. + raise ValueError( + f"store_service_not_managed: {store.capability}: service " + f"{capability.service!r} is not a managed engine" + ) + if store.sentinel.implied_engine is not engine: + raise ValueError( + f"sentinel_shape_mismatch: {store.capability}: sentinel implies " + f"{store.sentinel.implied_engine.value}, capability protocol resolves to " + f"{engine.value}" + ) + if store.baseline.strategy not in _ENGINE_STRATEGIES[engine]: + raise ValueError( + f"seed_strategy_unsupported: {store.capability}: {engine.value} does not " + f"support {store.baseline.strategy.value}" + ) + + if self.seed is not None: + # A store names its capability directly (no placeholder to resolve), so this half of + # the configuration_name rule is decidable here; the process-`environment` half needs + # placeholder scanning against the closed `{{...}}` vocabulary, which is §2e's job. + missing_name = [ + store.capability + for store in self.seed.stores + if not self.capabilities[store.capability].configuration_name + ] + if missing_name: + raise ValueError( + "configuration_name_required: " + ", ".join(sorted(set(missing_name))) + ) + + # v1's whole-manifest resolved-secret guard (`bundle.py`), reapplied here rather than + # dropped: v2 newly carries free-form `environment`/`build_environment` dicts, exactly + # where a resolved credential lands if an authoring stage ever inlines one instead of + # routing it through `secret_purposes`. `secret_purposes` itself is excluded from the + # dump — its key matches the secret-field pattern (`secret_...`) but it holds purpose + # identifiers, never values, the same reasoning v1 exempts `secret_refs` under. + _reject_secret_values( + self.model_dump(exclude={"digest": True, "processes": {"__all__": {"secret_purposes"}}}) + ) + return self + + +# --- §2c inputs_digest ------------------------------------------------------------------------ + + +def compute_inputs_digest( + root: str | Path, + migrations: Sequence[str], + seed_files: Sequence[str], + *, + engine: ManagedEngine, + version: str, +) -> str: + """The byte-exact `seed.stores[].baseline.inputs_digest` construction from §2c. + + sha256 over, for each file in ``migrations`` then ``seed_files`` **in listed order** (never + sorted — order is part of the identity, since migrations must apply in sequence), + ``\\n\\n``, followed by ``:\\n``. + Runs at authoring time, against ``migrations``/``seed_files`` as bundle-relative paths under + ``root`` (the bundle staging root) — the same paths the sealed manifest records, so the digest + is reproducible from the bundle's own field values. ``engine``/``version`` must be the store's + engine's own declared `ManagedProcess.engine`/`.version`, verbatim. + """ + root = Path(root) + digest = hashlib.sha256() + for relative_path in (*migrations, *seed_files): + _safe_relative(relative_path) + content = (root / relative_path).read_bytes() + digest.update(relative_path.encode("utf-8")) + digest.update(b"\n") + digest.update(str(len(content)).encode("utf-8")) + digest.update(b"\n") + digest.update(content) + digest.update(f"{engine.value}:{version}\n".encode("utf-8")) + return "sha256:" + digest.hexdigest() + + +# --- §2d bundle digest ------------------------------------------------------------------------ + + +def _canonical_json(value: dict[str, Any]) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode( + "utf-8" + ) + + +def seal_bundle_v2(manifest: EnvironmentBundleV2) -> str: + """The byte-exact `digest` construction from §2d (v1.7) — the single normative + implementation; producers call this, never reimplement it. + + sha256 over the canonical dump of the manifest with ``digest`` and ``files`` removed, then for + each ``files[]`` record, IN LISTED ORDER, the canonical dump of ``{path, sha256, size}`` + prefixed by its byte length as 8 bytes big-endian. "Canonical" = ``json.dumps(..., + sort_keys=True, separators=(",", ":"), ensure_ascii=False)``, both times. Operates on + `BundleFileV2` and `EnvironmentBundleV2.model_dump(mode="json")` directly — v1's `BundleFile` + never enters this construction, so a field added to v1's model cannot silently rekey a v2 + bundle's digest (F4, p4-round1-review). The hash covers the NORMALIZED dump, so adding an + optional field to `EnvironmentBundleV2` re-keys every previously sealed bundle — sealer and + verifier must ship together, which is exactly why there is only one implementation. + """ + core = manifest.model_dump(mode="json") + core.pop("digest", None) + core.pop("files", None) + digest = hashlib.sha256(_canonical_json(core)) + for record in manifest.files: + encoded = _canonical_json(record.model_dump(mode="json")) + digest.update(len(encoded).to_bytes(8, "big")) + digest.update(encoded) + return "sha256:" + digest.hexdigest() + + +def load_bundle_v2(path: str | Path) -> EnvironmentBundleV2: + """Parse and validate one `futureagi.environment-bundle.v2` manifest. + + ``path`` is either the manifest file itself or the bundle directory containing it. Schema + version is checked before full model validation runs, so a `…bundle.v1` manifest — or + anything else — is named explicitly rather than failing on an unrelated field. + """ + path = Path(path).expanduser() + target = path if path.is_file() else path / BUNDLE_V2_MANIFEST + if not target.is_file(): + raise BundleV2Error(f"bundle_manifest_missing: {target}") + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise BundleV2Error(f"bundle_manifest_invalid: {exc}") from exc + if not isinstance(raw, dict): + raise BundleV2Error("bundle_manifest_invalid: not a JSON object") + schema_version = raw.get("schema_version") + if schema_version != BUNDLE_V2_SCHEMA_VERSION: + raise BundleV2Error(f"bundle_schema_unsupported: {schema_version!r}") + try: + return EnvironmentBundleV2.model_validate(raw) + except ValidationError as exc: + raise BundleV2Error(f"bundle_manifest_invalid: {exc}") from exc + + +__all__ = [ + "BUNDLE_V2_MANIFEST", + "BUNDLE_V2_SCHEMA_VERSION", + "BaselineStrategy", + "BundleFileV2", + "BundleProvenanceV2", + "BundleRuntimeV2", + "BundleV2Error", + "CapabilityV2", + "EnvironmentBundleV2", + "EvidenceSeam", + "ManagedEngine", + "ManagedProcess", + "ProcessKind", + "ProcessUser", + "ReadinessProbeV2", + "RuntimeKindV2", + "Seed", + "SecretPurpose", + "Sentinel", + "SourceProcess", + "StartedCheck", + "StoreBaseline", + "StoreEntry", + "compute_inputs_digest", + "load_bundle_v2", + "seal_bundle_v2", +] diff --git a/src/fi/alk/harness/call_runner.py b/src/fi/alk/harness/call_runner.py new file mode 100644 index 00000000..276e253b --- /dev/null +++ b/src/fi/alk/harness/call_runner.py @@ -0,0 +1,730 @@ +"""The hosted lane's `CallRunner` — places one simulated LiveKit voice call and reports what +happened, satisfying `hosted_scheduler.CallRunner` exactly. + +Three sub-systems (world-handle-interface.md, hosted-execution-seams.md v1.15 §2a): + +1. **Placing the call.** The customer agent is already running INSIDE the Daytona sandbox, as a + world process the bundle's provisioner spawned (`process_runtime.py`) and registered with + LiveKit cloud under `LIVEKIT_AGENT_NAME=agent-w{WORLD_INDEX}`-style identity. This runner never + starts or manages that process — it drives `SimulationRunner` IN-PROCESS with a directly-built + `SimulationSpec` that dials the already-registered identity, mirroring `run/sdk_voice.py:: + build_spec` field-for-field but sourcing values from job config and the bundle's own scenario + document instead of `HARNESS_*` env vars (the local-only webhook/subprocess plumbing + `run/call.py`/`run/live.py` use is neither available nor appropriate in the guest). +2. **Collecting evidence.** The bundle declares exactly one `runtime.evidence_seam`: + `http_tool` or `tool_trace`. `http_tool` has NO guest-side capture surface anywhere in this + repo today (see `_collect_http_tool_calls`'s docstring — a verified finding, not an assumption) + and is intentionally left returning zero calls rather than inventing a capture proxy. + `tool_trace` is read from the world's own postgres database against an unpinned, isolated + convention (see `_collect_tool_trace_calls`'s docstring). Either way, zero calls captured is + never fabricated into something else — the scheduler's own `evidence_missing` retry-once policy + is the contract-correct handling for "no evidence." +3. **Uploading artifacts.** The transcript and any produced recordings are uploaded through the + adapter's `upload_artifact` (content-addressed, budget/level-gated, returns `None` on refusal — + never an exception) BEFORE this runner returns, so `CallOutcome.transcript_artifact`/ + `recording_artifacts` only ever carry ids the platform has already acked. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, Mapping, Protocol + +from fi import simulate +from fi.simulate.runtime import ( + AgentEndpointSpec, + EnvironmentSpec, + ExecutionPolicy, + SimulationSpec, + SimulatorPolicySpec, + TimeoutPolicy, + new_run_id, +) +from fi.simulate.runtime.report import SimulationReport +from fi.simulate.runtime.run import TestCaseStatus +from fi.simulate.runtime.runner import SimulationRunner + +from .bundle_v2 import EvidenceSeam +from .hosted_scheduler import CallAborted, CallOutcome +from .hosted_scheduler import Scenario as HostedScenario +from .job import HarnessJob +from .outbound import ArtifactKind, format_rfc3339_millis +from .process_runtime import EnvironmentRuntime +from .world.errors import WorldUnavailable +from .world.runtime import Call + +logger = logging.getLogger(__name__) + +# --- credential aliases / config keys a voice job must carry --- + +LIVEKIT_API_KEY_ALIAS = "LIVEKIT_API_KEY" +LIVEKIT_API_SECRET_ALIAS = "LIVEKIT_API_SECRET" +DEEPGRAM_API_KEY_ALIAS = "DEEPGRAM_API_KEY" +GEMINI_API_KEY_ALIAS = "GEMINI_API_KEY" +GOOGLE_API_KEY_ALIAS = "GOOGLE_API_KEY" # either this or GEMINI_API_KEY_ALIAS satisfies the LLM leg + +LIVEKIT_URL_CONFIG_KEY = "livekit_url" +CALL_TIMEOUT_CONFIG_KEY = "voice_call_timeout_seconds" + +_DEFAULT_CALL_TIMEOUT_SECONDS = 300.0 + +# sdk_voice.py::build_spec's own phase-overhead constants, reused verbatim so this runner's +# outer budget composes with the SDK's internal one the same way the local template does. +_CONNECT_TIMEOUT_SECONDS = 60.0 +_READINESS_TIMEOUT_SECONDS = 120.0 +_CLEANUP_TIMEOUT_SECONDS = 30.0 +_RUN_SECONDS_PAD_SECONDS = 60.0 +# Headroom beyond `spec.execution.timeout.run_seconds` -- SimulationRunner.run() already wraps +# `plugin.run(...)` in its OWN `asyncio.wait_for(..., timeout=spec.execution.timeout.run_seconds)` +# (runner.py) and catches that TimeoutError into a graceful `SimulationReport(status=TIMED_OUT)`. +# This runner's own outer wait_for must stay LARGER than that so the SDK's internal timeout fires +# first in the ordinary case; it only ever fires itself for a genuinely hung SDK (a real post-dial +# machinery failure) -- a runner-owned asyncio.wait_for as the last-resort bound. +_OUTER_WAIT_FOR_PAD_SECONDS = 60.0 + +# Unpinned by any contract and no producer exists yet. Isolated as one +# constant + two functions (`_clear_tool_trace_calls`, `_collect_tool_trace_calls`) so a real +# producer's disagreement on the name/shape is a one-line change. +_TOOL_TRACE_TABLE = "_alk_tool_trace" + +_RESULT_TRUNCATE_CHARS = 2000 + +# The real engine's zero-turn "agent joined but never spoke" failure codes (engines/livekit.py:: +# _conversation_outcome) -- see `_translate_report`'s `is_silent_agent` gate for why these two, and +# only at zero turns, get mapped to a normal CallOutcome instead of a CallAborted. +_SILENT_AGENT_FAILURE_CODES = frozenset({"no_conversation", "conversation_silence_timeout"}) + + +# --- collaborator seams (named, injectable test boundaries) ----------------------------------- + + +class ArtifactUploader(Protocol): + """Narrow slice of `hosted_entrypoint.OutboundAdapter` -- avoids importing that module here + (it imports THIS module's factory to wire the real CallRunner; importing it back would be + circular).""" + + async def upload_artifact( + self, + data: bytes, + *, + kind: ArtifactKind, + scenario_key: str | None = None, + deadline: float | None = None, + ) -> str | None: ... + + +PlaceCall = Callable[[SimulationSpec], Awaitable[SimulationReport]] + + +async def _default_place_call(spec: SimulationSpec) -> SimulationReport: + return await SimulationRunner().run(spec) + + +@dataclass(frozen=True) +class CallRunnerContext: + """Everything `hosted_entrypoint.py`'s `run_job` already has in scope by the wiring point + (~1662) that the real `CallRunnerImpl` needs but the bare `CallRunner` protocol signature + (`run(scenario, runtime)`) has no room to carry. Threaded through the EXTENDED + `build_call_runner(adapter, context)` seam.""" + + job: HarnessJob + bundle_dir: Path + work_directory: Path + evidence_seam: EvidenceSeam | None + target_provider_secret_values: Mapping[str, str] + attempt_number: int + + +# --- pre-dial validation ----------------------------------------------------------------------- + + +@dataclass(frozen=True) +class _MissingVoiceConfig: + aliases: tuple[str, ...] + config_keys: tuple[str, ...] + + def message(self) -> str: + parts = [] + if self.aliases: + parts.append("secrets=" + ",".join(self.aliases)) + if self.config_keys: + parts.append("config=" + ",".join(self.config_keys)) + return "voice_capability_unavailable: missing " + "; ".join(parts) + + +def _check_config( + job: HarnessJob, target_provider_secret_values: Mapping[str, str] +) -> _MissingVoiceConfig | None: + missing_aliases = [ + alias + for alias in (LIVEKIT_API_KEY_ALIAS, LIVEKIT_API_SECRET_ALIAS, DEEPGRAM_API_KEY_ALIAS) + if not target_provider_secret_values.get(alias) + ] + if not target_provider_secret_values.get( + GEMINI_API_KEY_ALIAS + ) and not target_provider_secret_values.get(GOOGLE_API_KEY_ALIAS): + missing_aliases.append(f"{GEMINI_API_KEY_ALIAS}_or_{GOOGLE_API_KEY_ALIAS}") + missing_config_keys = ( + [] if job.agent.config.get(LIVEKIT_URL_CONFIG_KEY) else [LIVEKIT_URL_CONFIG_KEY] + ) + if not missing_aliases and not missing_config_keys: + return None + return _MissingVoiceConfig(tuple(missing_aliases), tuple(missing_config_keys)) + + +def _dispatch_agent_name(runtime: EnvironmentRuntime) -> str | None: + """The ONLY place this repo reads the dispatch-identity metadata key, so a + change to the key name/convention is a one-line adapt. The provisioner + mirrors the agent process's rendered LIVEKIT_AGENT_NAME here; a bundle + that declares none (or an ambiguous set) leaves the key absent and the + caller's typed `CallAborted` below fires.""" + value = runtime.metadata.get("livekit_agent_name") + return value.strip() if isinstance(value, str) and value.strip() else None + + +# --- scenario document re-read (the _CompiledScenario the scheduler hands over carries no +# persona/instruction -- scenario_source.py:170-184's deliberately narrow Scenario-protocol +# shape) ------------------------------------------------------------------------------------ + + +class _ScenarioDocumentUnavailable(RuntimeError): + pass + + +def _read_scenario_document(bundle_dir: Path, scenario_key: str) -> dict[str, Any]: + """Re-reads `scenarios//scenario.json` from the bundle, matched by the document's OWN + `scenario_key` field -- never the folder name (`scenario_source.py`'s own convention; the two + are not guaranteed to match).""" + root = bundle_dir / "scenarios" + if not root.is_dir(): + raise _ScenarioDocumentUnavailable(f"no {root} directory in this bundle") + try: + children = sorted(root.iterdir()) + except OSError as exc: + raise _ScenarioDocumentUnavailable(f"cannot list {root}: {exc}") from exc + for child in children: + if not child.is_dir(): + continue + doc_path = child / "scenario.json" + if not doc_path.is_file(): + continue + try: + body = json.loads(doc_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + continue + if isinstance(body, dict) and body.get("scenario_key") == scenario_key: + instruction = body.get("instruction") + if not isinstance(instruction, str) or not instruction.strip(): + raise _ScenarioDocumentUnavailable( + f"{child.name}/scenario.json has no non-empty instruction" + ) + return body + raise _ScenarioDocumentUnavailable( + f"no scenario.json under {root} carries scenario_key={scenario_key!r}" + ) + + +# --- deterministic room naming (asserted verbatim by tests/harness/test_call_runner.py). WHY this +# is a PREFIX guarantee, not a full-match one: in managed room_mode, engines/livekit.py:: +# _resolve_room_name appends its own `-{invocation_id}-{test_case_id[-12:]}` suffix unless +# `room_name_verbatim` is set (which this runner does not set) -- the scheme below still gives +# every call a unique, deterministic, greppable prefix; only the exact wire-level name is not this +# string verbatim. ------------------------------------------------------------------------------- + + +def _room_name( + *, job_id: str, attempt_number: int, scenario_key: str, scenario_attempt: int +) -> str: + return f"harness-{job_id[:8]}-a{attempt_number}-{scenario_key}-s{scenario_attempt}" + + +def _duration_ms(started_at: datetime, ended_at: datetime) -> int: + return max(0, int((ended_at - started_at).total_seconds() * 1000)) + + +# --- SimulationSpec construction (mirrors run/sdk_voice.py::build_spec field-for-field; values +# come from job config / the re-read scenario document instead of HARNESS_* env vars) --------- + + +def _simulator_definition() -> Any: + """Mirrors `sdk_voice.py::_simulator()`'s own shipped defaults + exactly (google/gemini LLM, deepgram STT+TTS) -- the only provider combination this runner's + pre-dial credential check (`_check_config`) validates.""" + return simulate.SimulatorAgentDefinition( + llm={"provider": "google", "model": "gemini-2.5-flash-lite", "temperature": 0.35}, + stt={"provider": "deepgram", "model": "nova-2", "language": "en"}, + tts={"provider": "deepgram", "model": "aura-asteria-en", "voice": "aura-asteria-en"}, + instructions=( + "Act as the customer described by the scenario. Speak naturally and briefly. " + "Use only the supplied facts and never invent account, address, payment, or " + "verification data. Do not volunteer private data: agree when asked whether a " + "verification code should be sent, and disclose the actual code only after the " + "agent says it was sent and explicitly asks you to read it. Answer repair questions " + "with the missing fact, not by restarting the request. Never repeat the same answer " + "more than twice. When the requested outcome is complete, thank the agent and end " + "the call." + ), + allow_interruptions=True, + ) + + +def _scenario_spec(doc: Mapping[str, Any]) -> Any: + """Mirrors `sdk_voice.py::_scenario()`'s own transformation exactly, sourced from the re-read + scenario document instead of `HARNESS_*` env vars.""" + fixture = doc.get("fixture") if isinstance(doc.get("fixture"), dict) else {} + persona = dict(doc.get("persona") or {}) + persona["role"] = "customer" + metadata = dict(persona.get("metadata") or {}) + if fixture.get("phone"): + metadata["caller_phone"] = str(fixture["phone"]) + persona["metadata"] = metadata + knowledge = [ + { + "key": str(key), + "value": json.dumps(value, ensure_ascii=False, default=str), + "disclosure": "on_request", + } + for key, value in fixture.items() + if key != "origin" + ] + persona_model = simulate.Persona( + persona=persona, + situation=doc["instruction"], + outcome=(doc.get("tests") or "Complete the requested task and close naturally."), + knowledge=knowledge, + behavior_policy={ + "disclosure_policy": 0.72, + "cooperation_bounds": 0.9, + "repair_propensity": 0.85, + }, + ) + return simulate.Scenario( + name=str(doc.get("scenario_key") or doc.get("name") or "harness-voice"), + dataset=[persona_model], + ) + + +def _build_spec( + *, + run_id: str, + room_name: str, + agent_name: str, + doc: Mapping[str, Any], + livekit_url: str, + call_timeout_seconds: float, + run_seconds: float, + recordings_root: Path, +) -> SimulationSpec: + recording_dir = recordings_root / run_id / "recordings" + params = { + "record_audio": True, + "recording_root": str(recording_dir), + "recording_case_directory": str(recording_dir), + "min_turn_messages": 6, + "max_seconds": call_timeout_seconds, + "connect_timeout": _CONNECT_TIMEOUT_SECONDS, + "readiness_timeout": _READINESS_TIMEOUT_SECONDS, + "cleanup_timeout": _CLEANUP_TIMEOUT_SECONDS, + "conversation_direction": "agent_first", + "agent_first_silence_timeout_seconds": 45.0, + } + agent = simulate.AgentDefinition( + name="harness-livekit-target", + agent_name=agent_name, + system_prompt=doc["instruction"], + transport={"kind": "webrtc"}, + ) + runtime_spec = simulate.LiveKitSimulatorRuntime( + url=livekit_url, room_name=room_name, room_mode="managed", + ) + return SimulationSpec( + run_id=run_id, + environment=EnvironmentSpec( + adapter="voice", + world_kind="voice_telephony", + config={ + "agent_definition": agent.model_dump(mode="json", exclude_none=True), + "livekit_runtime": runtime_spec.model_dump(mode="json", exclude_none=True), + "simulator": _simulator_definition().model_dump(mode="json", exclude_none=True), + "params": params, + }, + ), + target=AgentEndpointSpec(adapter="webrtc"), + simulator=SimulatorPolicySpec(adapter="livekit_simulator"), + scenario=_scenario_spec(doc), + execution=ExecutionPolicy( + direction="agent_first", timeout=TimeoutPolicy(run_seconds=run_seconds) + ), + ) + + +# --- evidence collection ----------------------------------------------------------------------- + + +def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> Any | None: + """Protocol-based lookup, matching `hosted_entrypoint.py::_find_postgres_endpoint`'s own + already-correct convention -- capability slugs are bundle-author-chosen (`build_endpoints`, + process_runtime.py:318-339), never a fixed key, so a hardcoded `endpoints["database"]` would + break for any bundle that names its capability slug differently. Re-implemented locally rather + than imported: importing from `hosted_entrypoint.py` here would be circular (it imports this + module's factory).""" + for endpoint in runtime.endpoints.values(): + if endpoint.protocol == "postgres": + return endpoint + return None + + +def _collect_http_tool_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: + """No guest-side capture surface exists anywhere in this repo for the + `http_tool` evidence seam. Verified, not assumed: `world/handle.py::HostedWorld.call()` + raises `WorldUnavailable` unconditionally with a docstring stating the wire format "is not + pinned anywhere in the contracts yet"; `process_runtime.py`'s own `provision()` signature + comment says "evidence-seam wiring is out of this phase's scope"; no `TOOLS_API_URL` wiring + exists in the hosted lane at all (the local lane's `ProvisionedWorld`/`TOOLS_API_URL` mechanism + lives in `provision.py`/`world/provisioned.py`, out of scope here and inapplicable to the guest + regardless). Deliberately stopped rather than inventing a capture proxy: a job whose bundle + declares `evidence_seam: http_tool` reads zero calls every time, which the scheduler's own + `evidence_missing` retry-once policy turns into the correct, honest outcome -- never a crash, + never fabricated evidence.""" + del runtime + return () + + +def _clear_tool_trace_calls(dsn: str) -> None: + """world-handle-interface.md: "setup's tool calls are NOT evidence (the runner clears them + before the call starts, as the local runner does)" -- the local runner's analog is + `world.calls = []` right before dialing (`run/simulation.py`). Best-effort: a missing table (no + producer yet) or any connection error is swallowed, never raised. Clearing + is housekeeping, not a correctness requirement, while nothing writes this table yet; once a + real producer lands this stops being a no-op automatically.""" + try: + import psycopg + + with psycopg.connect(dsn, autocommit=True, connect_timeout=5) as connection: + connection.execute(f'DELETE FROM "{_TOOL_TRACE_TABLE}"') # noqa: S608 - fixed identifier, no interpolated user input + except Exception as exc: # noqa: BLE001 - best-effort housekeeping only, never a call-blocking failure + # WHY: never log exc_info / str(exc) here -- a psycopg connection failure embeds the raw + # DSN (including the world DB password) in its own exception message; only the exception + # TYPE is safe for a local log line. + logger.debug("tool_trace clear skipped (table likely absent): %s", type(exc).__name__) + + +def _collect_tool_trace_calls(runtime: EnvironmentRuntime) -> tuple[Call, ...]: + """`_alk_tool_trace`'s name and column shape are an isolated local + convention -- unpinned by any contract (the only harness-reserved table anywhere in this + repo is `_alk_conformance`, unrelated), no producer exists yet. Isolated in this one function + (+ `_clear_tool_trace_calls`) so a real producer's disagreement on the name/shape is a one-line + change. Any failure (missing table, connection refused, malformed row) degrades to `()` -- + never a crash, never fabricated evidence, matching `_collect_http_tool_calls`'s stopped + behavior above.""" + endpoint = _find_postgres_endpoint(runtime) + if endpoint is None: + return () + try: + import psycopg + + with psycopg.connect( + endpoint.address, + autocommit=True, + connect_timeout=5, + options="-c default_transaction_read_only=on", + ) as connection: + cursor = connection.execute( + f'SELECT name, arguments, result, ok, error, at FROM "{_TOOL_TRACE_TABLE}" ' # noqa: S608 + "ORDER BY at ASC" + ) + rows = cursor.fetchall() + columns = [description[0] for description in cursor.description or []] + except Exception as exc: # noqa: BLE001 - missing table / connection failure -> no evidence, not a crash + # WHY: same DSN-in-exception-message risk as `_clear_tool_trace_calls` above -- log only + # the exception TYPE, never exc_info/str(exc), which can carry the world DB password. + logger.debug("tool_trace read failed; treating as no evidence: %s", type(exc).__name__) + return () + + calls: list[Call] = [] + for row in rows: + record = dict(zip(columns, row, strict=True)) + name = record.get("name") + if not isinstance(name, str) or not name: + continue + arguments = record.get("arguments") + if not isinstance(arguments, dict): + arguments = {} + ok = bool(record.get("ok", True)) + raw_result = record.get("result") + if isinstance(raw_result, str): + result: Any = _truncate(raw_result) + else: + # Already parsed JSON (dict/list/etc, psycopg's own jsonb decoding) -- per + # world-handle-interface.md, only the STRING form is truncated at 2000 chars. + result = raw_result + error = _truncate(str(record.get("error") or "")) + raw_at = record.get("at") + at = float(raw_at) if isinstance(raw_at, (int, float)) else 0.0 + calls.append( + Call(name=name, arguments=arguments, result=result, ok=ok, error=error, refused=not ok, at=at) + ) + return tuple(calls) + + +def _truncate(value: str, *, limit: int = _RESULT_TRUNCATE_CHARS) -> str: + return value if len(value) <= limit else value[:limit] + + +# --- the runner ---------------------------------------------------------------------------- + + +class CallRunnerImpl: + """Satisfies `hosted_scheduler.CallRunner`. See the module docstring for the three + sub-systems this class implements.""" + + def __init__( + self, + adapter: ArtifactUploader, + context: CallRunnerContext, + *, + place_call: PlaceCall | None = None, + environ: dict[str, str] | None = None, + ) -> None: + self._adapter = adapter + self._context = context + self._place_call = place_call or _default_place_call + # WHY: the underlying LiveKit engine reads these directly via `os.environ.get(...)` deep + # inside `engines/livekit.py` / `livekit_models.py` -- they are NOT `SimulationSpec` + # fields, so there is no other way to hand them over. Exported ONCE here, at construction, + # not per-call: the values are job-level (the same secret for every scenario/attempt on + # this job) and W>1 means each world's CallRunner.run() executes inside this SAME guest + # process but against a per-world SANDBOXED agent process reached over the network -- no + # other in-process worker ever races this write, so one job-level export is race-free. + target_environ = os.environ if environ is None else environ + for alias in ( + LIVEKIT_API_KEY_ALIAS, + LIVEKIT_API_SECRET_ALIAS, + DEEPGRAM_API_KEY_ALIAS, + GEMINI_API_KEY_ALIAS, + GOOGLE_API_KEY_ALIAS, + ): + value = context.target_provider_secret_values.get(alias) + if value: + target_environ[alias] = value + self._missing_config = _check_config(context.job, context.target_provider_secret_values) + self._scenario_attempt_counts: dict[str, int] = {} + + async def run(self, scenario: HostedScenario, runtime: EnvironmentRuntime) -> CallOutcome: + if self._missing_config is not None: + # Pre-dial: dialing never starts, so no partial -- and never `WorldUnavailable` (that + # code is reserved by the contract for a world-level capability mismatch, not a + # job-level voice config gap). + raise CallAborted(self._missing_config.message()) + + agent_name = _dispatch_agent_name(runtime) + if agent_name is None: + raise CallAborted( + "voice_dispatch_identity_unavailable: runtime.metadata['livekit_agent_name'] is " + f"not set for world {runtime.world_index}" + ) + + try: + doc = _read_scenario_document(self._context.bundle_dir, scenario.scenario_key) + except _ScenarioDocumentUnavailable as exc: + raise CallAborted(f"voice_scenario_document_unavailable: {exc}") from exc + + scenario_attempt = self._scenario_attempt_counts.get(scenario.scenario_key, 0) + 1 + self._scenario_attempt_counts[scenario.scenario_key] = scenario_attempt + room_name = _room_name( + job_id=self._context.job.job_id, + attempt_number=self._context.attempt_number, + scenario_key=scenario.scenario_key, + scenario_attempt=scenario_attempt, + ) + + raw_timeout = self._context.job.agent.config.get(CALL_TIMEOUT_CONFIG_KEY) + call_timeout_seconds = ( + float(raw_timeout) if isinstance(raw_timeout, (int, float)) else _DEFAULT_CALL_TIMEOUT_SECONDS + ) + run_seconds = ( + call_timeout_seconds + + _CONNECT_TIMEOUT_SECONDS + + _READINESS_TIMEOUT_SECONDS + + _CLEANUP_TIMEOUT_SECONDS + + _RUN_SECONDS_PAD_SECONDS + ) + + spec = _build_spec( + run_id=new_run_id(), + room_name=room_name, + agent_name=agent_name, + doc=doc, + livekit_url=str(self._context.job.agent.config.get(LIVEKIT_URL_CONFIG_KEY)), + call_timeout_seconds=call_timeout_seconds, + run_seconds=run_seconds, + recordings_root=self._context.work_directory / "voice-calls", + ) + + if self._context.evidence_seam is EvidenceSeam.TOOL_TRACE: + endpoint = _find_postgres_endpoint(runtime) + if endpoint is not None: + _clear_tool_trace_calls(endpoint.address) + + started_at = datetime.now(timezone.utc) + outer_timeout = run_seconds + _OUTER_WAIT_FOR_PAD_SECONDS + try: + report = await asyncio.wait_for(self._place_call(spec), timeout=outer_timeout) + except asyncio.CancelledError: + raise + except asyncio.TimeoutError as exc: + raise CallAborted( + "voice_call_runner_timeout: place_call exceeded its outer budget " + f"({outer_timeout:.0f}s)", + partial=self._timing_only_outcome(started_at), + ) from exc + except Exception as exc: # noqa: BLE001 - post-dial machinery failure, never let it escape raw + raise CallAborted( + f"voice_call_runner_crashed: {type(exc).__name__}: {exc}", + partial=self._timing_only_outcome(started_at), + ) from exc + + try: + return await self._translate_report( + report, runtime=runtime, scenario_key=scenario.scenario_key, started_at=started_at + ) + except (CallAborted, WorldUnavailable): + # `_translate_report`'s own typed control-flow (non-completed status, no test case, + # agent-never-joined) -- never re-wrap an intentional abort. + raise + except Exception as exc: # noqa: BLE001 - a transcript/recording read or upload surprise + # must never lose the timing this call already measured (the receipt's `call` field + # must not be null once the call has genuinely started) by escaping run() raw. + raise CallAborted( + f"voice_call_translate_crashed: {type(exc).__name__}: {exc}", + partial=self._timing_only_outcome(started_at), + ) from exc + + def _timing_only_outcome(self, started_at: datetime) -> CallOutcome: + ended_at = datetime.now(timezone.utc) + return CallOutcome( + calls=(), + turns=0, + started_at=format_rfc3339_millis(started_at), + ended_at=format_rfc3339_millis(ended_at), + duration_ms=_duration_ms(started_at, ended_at), + ) + + async def _translate_report( + self, + report: SimulationReport, + *, + runtime: EnvironmentRuntime, + scenario_key: str, + started_at: datetime, + ) -> CallOutcome: + case = report.test_cases[0] if report.test_cases else None + case_started_at = ( + case.started_at if case is not None and case.started_at is not None else started_at + ) + ended_at = ( + case.ended_at + if case is not None and case.ended_at is not None + else datetime.now(timezone.utc) + ) + turns = len(case.result.messages) if case is not None and case.result is not None else 0 + + transcript_artifact: str | None = None + recording_artifacts: list[str] = [] + if case is not None and case.result is not None: + result = case.result + if result.transcript: + transcript_artifact = await self._adapter.upload_artifact( + result.transcript.encode("utf-8"), + kind=ArtifactKind.TRANSCRIPT, + scenario_key=scenario_key, + ) + for path_str, kind in ( + (result.audio_combined_path, ArtifactKind.RECORDING_COMBINED), + (result.audio_stereo_path, ArtifactKind.RECORDING_STEREO), + (result.audio_input_path, ArtifactKind.RECORDING_CUSTOMER), + (result.audio_output_path, ArtifactKind.RECORDING_ASSISTANT), + ): + if not path_str: + continue + path = Path(path_str) + if not path.is_file(): + continue + artifact_id = await self._adapter.upload_artifact( + path.read_bytes(), kind=kind, scenario_key=scenario_key, + ) + if artifact_id is not None: + recording_artifacts.append(artifact_id) + + base = CallOutcome( + calls=(), + turns=turns, + started_at=format_rfc3339_millis(case_started_at), + ended_at=format_rfc3339_millis(ended_at), + duration_ms=_duration_ms(case_started_at, ended_at), + transcript_artifact=transcript_artifact, + recording_artifacts=tuple(recording_artifacts), + ) + + if case is None: + raise CallAborted("voice_call_no_test_case: SimulationReport carried no test case", partial=base) + + if case.status is TestCaseStatus.AGENT_UNAVAILABLE: + # world-handle-interface.md: "the agent never joined" is a WORLD failure, not a + # scenario one -- the agent is part of the world, so the scheduler retires it and + # retries elsewhere. Verified against the engine's own source (engines/livekit.py): + # this status fires ONLY on a readiness-stage timeout with a session already started + # but no target dispatched -- exactly "dispatch fails, agent never joins," never a + # mid-call condition. + reason = case.failure.message if case.failure is not None else "agent_unavailable" + raise WorldUnavailable(f"target agent never joined the room: {reason}") + + # A genuinely silent agent-first call (agent joined, zero conversational turns) reaches + # the real engine (engines/livekit.py::_conversation_outcome) as FAILED with code + # "no_conversation" or "conversation_silence_timeout" and zero messages -- never as a + # COMPLETED case with zero turns (COMPLETED requires >= min_turn_messages AND role + # alternation, so the engine cannot produce that shape). Scoped to zero turns only: a + # short-but-nonzero conversation on either code still failed the completion bar for a real + # reason and must stay a CallAborted below. + is_silent_agent = ( + case.status is TestCaseStatus.FAILED + and turns == 0 + and case.failure is not None + and case.failure.code in _SILENT_AGENT_FAILURE_CODES + ) + + if case.status is not TestCaseStatus.COMPLETED and not is_silent_agent: + reason = case.failure.message if case.failure is not None else case.status.value + raise CallAborted(f"voice_call_not_completed: {case.status.value}: {reason}", partial=base) + + # Never fabricate calls for a call that produced no conversation -- the scheduler's own + # coverage guarantee turns an empty `calls` tuple into evidence_missing/simulator + # regardless of turns (hosted_scheduler.py's own unconditioned-on-turns rule). + calls = () if is_silent_agent else self._collect_calls(runtime) + return CallOutcome( + calls=calls, + turns=base.turns, + started_at=base.started_at, + ended_at=base.ended_at, + duration_ms=base.duration_ms, + transcript_artifact=base.transcript_artifact, + recording_artifacts=base.recording_artifacts, + ) + + def _collect_calls(self, runtime: EnvironmentRuntime) -> tuple[Call, ...]: + seam = self._context.evidence_seam + if seam is EvidenceSeam.HTTP_TOOL: + return _collect_http_tool_calls(runtime) + if seam is EvidenceSeam.TOOL_TRACE: + return _collect_tool_trace_calls(runtime) + # Unrecognized/None (should not happen for a `kind: process` bundle past preflight -- + # bundle_v2.py requires `evidence_seam` whenever `kind is PROCESS` -- but degrading rather + # than crashing keeps this on the scheduler's own evidence_missing path, never a raw + # exception). + return () diff --git a/src/fi/alk/harness/checks.py b/src/fi/alk/harness/checks.py index f17d2e41..8e4c26e0 100644 --- a/src/fi/alk/harness/checks.py +++ b/src/fi/alk/harness/checks.py @@ -69,13 +69,23 @@ def run_check( return Outcome( name, False, - f"the check raised {type(failed).__name__}: {failed}", + f"the check raised {type(failed).__name__}: {str(failed)[:200]}", broken=True, ) - if said is None or said is True: + if said is None or said is True or (isinstance(said, str) and not said.strip()): return Outcome(name, True) - return Outcome(name, False, str(said) if said is not True else "") + if said is False: + return Outcome(name, False, "False") + if not isinstance(said, str): + return Outcome( + name, + False, + f"the check returned {type(said).__name__} {repr(said)[:200]}; a check returns a " + "sentence naming what is wrong, or None when it held.", + broken=True, + ) + return Outcome(name, False, said) def all_held(outcomes: Sequence[Outcome]) -> bool: @@ -120,7 +130,7 @@ def run_world_check( return Outcome( name, False, - f"the check raised {type(failed).__name__}: {failed}", + f"the check raised {type(failed).__name__}: {str(failed)[:200]}", broken=True, ) return Outcome(name, said is None, "" if said is None else str(said)) diff --git a/src/fi/alk/harness/folder.py b/src/fi/alk/harness/folder.py index 2b5838ad..fa6fec93 100644 --- a/src/fi/alk/harness/folder.py +++ b/src/fi/alk/harness/folder.py @@ -54,8 +54,9 @@ if len(_sys.argv) > 2: _calls = [_Call(**_one) for _one in _json.loads(_Path(_sys.argv[2]).read_text())] _said = check(_world, _calls) - print("held" if _said is None else f"FAILED: {_said}") - raise SystemExit(0 if _said is None else 1) + _held = _said is None or _said is True or (isinstance(_said, str) and not _said.strip()) + print("held" if _held else f"FAILED: {_said}") + raise SystemExit(0 if _held else 1) """ @@ -99,10 +100,23 @@ def _run(source: str, name: str, entry: str, *args: Any) -> Outcome: if said is None or said is True or (isinstance(said, str) and not said.strip()): return Outcome(True) if said is False: + # Bare False from ready() names no precondition, so a scenario that hits it cannot be + # told apart from one whose ready.py is simply wrong — that is our mistake, not a + # generation-time precondition failure, so ready() alone reports it broken. return Outcome( False, f"{name} returned False without saying what is wrong. Return the sentence instead, " "or None if it holds.", + broken=(entry == "ready"), + ) + if entry == "ready" and not isinstance(said, str): + # Same reasoning as bare False, widened: ready() has no way to turn a non-string value + # into a precondition sentence, so any of them is our mistake rather than the world's. + return Outcome( + False, + f"{name} returned {type(said).__name__} {repr(said)[:200]}. Return the sentence " + "naming what is missing, or None if it holds.", + broken=True, ) return Outcome(False, str(said)) diff --git a/src/fi/alk/harness/hosted_entrypoint.py b/src/fi/alk/harness/hosted_entrypoint.py index 2a4f792a..3f93f919 100644 --- a/src/fi/alk/harness/hosted_entrypoint.py +++ b/src/fi/alk/harness/hosted_entrypoint.py @@ -1,41 +1,1974 @@ -"""Standalone entrypoint run inside one isolated hosted ALK sandbox. +"""The hosted guest's `main()` — `hosted-execution-seams.md` v1.14 §0/§4/§5, `outbound-channels.md` +v1.3, `world-handle-interface.md` v3.4. Everything between "sandbox starts" and "exit code": read +`/work/job.json`, load the platform capability file, run §2e preflight, pre-allocate scenarios +against `endpoints.scenarios`, provision the world pool, drive the scenario loop, adapt its events/ +receipts/artifacts onto the real outbound clients, and honor the exit-code contract (§0.6). -The platform creates the typed job and prepares a source checkout through its repository -integration. It does not execute harness stages. This process consumes those inputs and runs the -same ``HarnessExecutor`` used locally. +Ownership boundary (read this before touching orchestration order): the stages BEFORE bundle +authoring — `understanding_agent`, `generating_environment`, `building_environment`'s bundle-write +half — belong to Rishav's stages (contract §6) and are not implemented anywhere in this repo yet. +This module does not attempt them. `BundleSource`/`ScenarioSource` below are the seams a later +change wires the real stages through; until then their defaults raise a typed, clearly-named error +rather than silently producing a fake bundle or a fake scenario set. + +`process_runtime.py`, `hosted_scheduler.py`, and `outbound.py` were being fixed by parallel workers +while this module was written. It codes against the four frozen contracts and the cross-review +obligation lists, not against those files' exact HEAD. """ from __future__ import annotations import argparse import asyncio +import hashlib +import json +import logging +import random +import signal +import time +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timezone from pathlib import Path +from typing import Any, Callable, Protocol, Sequence + +from . import outbound as ob +from .bundle_v2 import BundleV2Error, EnvironmentBundleV2, load_bundle_v2 +from .call_runner import CallRunnerContext, CallRunnerImpl +from .hosted_scheduler import ( + CallOutcome, + CallRunner, + HostedScheduler, + ResultReceipt, + RunResult, + Scenario, + World, + WorldFactory, + WorldPool, + WorldProvisioner, +) +from .job import ( + ArtifactLevel, + ExecutionMode, + FailureDomain, + HarnessArtifactPolicy, + HarnessJob, + HarnessStage, +) +from .process_preflight import PreflightError, preflight_bundle +from .process_runtime import ( + SECTION_2F_DOMAIN, + EnvironmentRuntime, + ProcessRuntimeError, + ProcessRuntimeProvider, + RuntimeEndpoint, +) +from .scenario_source import BundleScenarioSource, ScenarioDocumentInvalid, bundle_has_scenarios +from .world.errors import WorldStoreless +from .world.handle import HostedWorld +from .world.stores.postgres import AttachedPostgresStore + +logger = logging.getLogger(__name__) + +# --- §0.6 exit-code contract -------------------------------------------------------------------- +# +# 0 = any terminal stage reached (completed/failed/canceled), outbox flushed. 3 = fenced/superseded +# (HostedFencedError anywhere -> stop emitting, no terminal event, exit 3). 4 = the terminal was +# decided but the final drain could not deliver it (the events channel failed, or the platform +# permanently rejected the terminal item itself) -- the gateway treats it exactly like a crash +# (infrastructure retry, fresh channels), but the distinct code tells operators the job DID reach a +# terminal state, unlike a genuine crash. Any other non-zero = the guest crashed before a terminal +# state -- the gateway records `infrastructure`. Capabilities-file failures are explicitly carved +# out of the "any other non-zero" bucket only by CODE (they must never be 3, per +# outbound-channels.md v1.3's rejection table); they still use a non-zero exit here since there is +# no channel to report a terminal FAILED event through. +EXIT_OK = 0 +EXIT_FENCED = 3 +EXIT_TERMINAL_UNDELIVERED = 4 # terminal reached but not provably flushed on the final drain. +EXIT_BOOT_FAILURE = 1 # capabilities.json could not be loaded -- no channel, no event (v1.3 table). +EXIT_CRASHED = 2 # an uncaught failure before any terminal stage was reached. + +# Cancellation signal (spine §0 step 7 / outbound-channels.md "Cancellation signal"). The task +# brief that spawned this module named `/work/cancel.json`; the two frozen contracts that actually +# define this file (seams §0 step 7, outbound-channels "Cancellation signal") both name +# `/run/futureagi/cancel.json`. Contracts are authoritative over a task brief. +CANCEL_SIGNAL_PATH = "/run/futureagi/cancel.json" + +# STUCK DECISION (fail-safe/reversible; contract gap): the invocation contract +# (spine §0 step 5) pins the entrypoint's argv to exactly `job --source ... --output ...`; `--output` +# is `/work/artifacts` (spine layout block), so `work_directory` (what `preflight_bundle`/ +# `provision`/`write_build_output` all want -- the `/work` root) is derived as `output.parent` +# rather than taken as a separate flag, since the frozen invocation line has no room for one. +# `bundle_dir` has no convention anywhere in the frozen documents at all (bundle authoring is not +# built yet); `DEFAULT_BUNDLE_DIR_NAME` is this module's own placeholder location, overridable via +# `BundleSource` injection so a later change can point it at wherever the real authoring stage ends +# up writing without touching this file's orchestration. +DEFAULT_BUNDLE_DIR_NAME = "bundle" +EVENTS_SPOOL_DIR_NAME = "outbound-spool" # must not live under work_directory/"artifacts". + +SECRETS_PATH = Path("/run/futureagi/secrets.json") -from .executor import HarnessExecutor -from .job import ExecutionMode, HarnessJob, HarnessStage +# ================================================================================================= +# Boot -- job.json + capabilities.json (§0.2/§0.4; outbound-channels.md Authentication). +# ================================================================================================= -async def _run(job_path: Path, source: Path, output: Path) -> int: + +def load_job(job_path: Path) -> HarnessJob: + """§0.2: `/work/job.json` is the provisioner's job-identity and configuration source.""" job = HarnessJob.model_validate_json(job_path.read_text(encoding="utf-8")) if job.execution is not ExecutionMode.HOSTED: raise ValueError("hosted_entrypoint_requires_hosted_job") - status = await HarnessExecutor().run(job, source=source, output=output) - print(status.model_dump_json(exclude_none=True), flush=True) - return 0 if status.stage is HarnessStage.COMPLETED else 1 + return job -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(prog="alk-harness-worker") - parser.add_argument("job", type=Path, help="typed HarnessJob JSON") - parser.add_argument( - "--source", required=True, type=Path, help="sandbox-owned source checkout" +def resolve_parallelism(job: HarnessJob) -> int: + """`job.runtime.parallelism` = W (glossary). Returns the RAW requested value, never + clamped -- §2e.7 reserves `parallelism_out_of_range` for a W outside 1..8, and + `preflight_bundle` (called BEFORE any provisioning) is the enforcement point for the UPPER + bound. The lower bound never reaches preflight at all: `RuntimeRequirements.parallelism`'s own + `ge=1` rejects a non-positive W earlier, at `load_job`, as a deliberate defense-in-depth floor + (harmless today since the gateway caps W at admission before a job is ever built). Clamping + here would silently launder an in-range-but-wrong W and make `parallelism_out_of_range` + permanently unreachable for the upper bound.""" + return job.runtime.parallelism + + +def job_secret_purposes(job: HarnessJob) -> dict[str, str]: + """§1: `agent.secret_refs` alias -> `SecretRef.purpose`, the shape `preflight_bundle` wants.""" + return {alias: ref.purpose for alias, ref in job.agent.secret_refs.items()} + + +def peek_secret_values(secrets_path: Path) -> tuple[str, ...]: + """A non-destructive read of `/run/futureagi/secrets.json`'s VALUES ONLY, for outbound + redaction (`extra_secret_values` — outbound.py's `redact_outbound_text`). §0.3's lifetime rule + ("the provisioner loads this file into memory at startup and deletes it") is honored by + `ProcessRuntimeProvider` itself; this is an additional, side-effect-free read (no unlink) done + once at boot so free-text event/log/failure fields can be scrubbed of every resolved secret + value, not just URL userinfo. Never fatal: a missing/malformed file just means no extra values + to scrub, matching `redact_outbound_text`'s own `extra_secret_values=()` default.""" + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return () + if not isinstance(raw, dict): + return () + return tuple(str(value) for value in raw.values() if value) + + +def peek_target_provider_secret_values( + secrets_path: Path, secret_purposes: dict[str, str] +) -> dict[str, str]: + """The same non-destructive, no-unlink read as `peek_secret_values` (same file, same timing + constraint -- called BEFORE `pool.start()`, which is what actually deletes the file), but + ALIAS-preserving and filtered to `purpose: target_provider` -- `peek_secret_values` throws the + alias away, which is fine for outbound redaction (it only needs the raw values) but useless for + the real `CallRunner`, which needs to pick e.g. `LIVEKIT_API_KEY` out of the map by name. Never + fatal: a missing/malformed file just means no target-provider secrets are available yet, + matching `CallRunnerImpl`'s own pre-dial validation (it reports the gap as a typed + `CallAborted`, never crashes on an empty map).""" + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + if not isinstance(raw, dict): + return {} + return { + str(alias): str(value) + for alias, value in raw.items() + if secret_purposes.get(str(alias)) == "target_provider" + } + + +# ================================================================================================= +# Bundle source -- §2 bundle authoring is not this module's (or built anywhere yet); injectable. +# ================================================================================================= + + +class BundleUnavailableError(RuntimeError): + """Raised by a `BundleSource` when no bundle could be produced/located. Mapped the same way as + a `PreflightError` (FAILED, `FailureDomain.ENVIRONMENT`, stage `validating_environment`) — + from the entrypoint's point of view "no bundle" and "bad bundle" are the same class of + environment-authoring fault, and §2e's own failure table has no separate code for it.""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +class BundleSource(Protocol): + def load( + self, job: HarnessJob, *, source: Path, work_directory: Path + ) -> tuple[EnvironmentBundleV2, Path]: ... + + +# §2e's closed failure-code table (hosted-execution-seams.md) -- `BundleV2Error` has no typed +# `.code` (a bare `RuntimeError`), so `DefaultBundleSource.load` below string-splits +# its message on ":". `bundle_manifest_missing` (one of the four messages `load_bundle_v2` can +# raise) is not in this table -- a real contract gap -- so both that code AND anything else the +# split produces outside this frozen set fall back to `bundle_manifest_invalid` rather than +# shipping an unlisted code across the outbound seam. +_SECTION_2E_CODES = frozenset( + { + "compose_not_hosted", "engine_unsupported", "no_sql_store", "seed_missing", + "seed_strategy_unsupported", "sentinel_shape_mismatch", "store_protocol_unsupported", + "capability_engine_mismatch", "store_service_not_managed", "reserved_name", + "unknown_placeholder", "unknown_field", "secret_in_bundle", "secret_unclaimed", + "secret_missing", "build_requires_root", "user_assignment_invalid", + "configuration_name_duplicate", "configuration_name_required", + "configuration_name_reserved", "sentinel_shape_invalid", "capability_unresolved", + "service_unresolved", "control_service_unresolved", "process_name_duplicate", + "inputs_digest_mismatch", "bundle_schema_unsupported", "bundle_manifest_invalid", + "bundle_manifest_drifted", "bundle_digest_mismatch", "bundle_digest_invalid", + "inputs_digest_invalid", "file_sha256_invalid", "source_digest_invalid", + "bundle_file_missing", "bundle_file_changed", "bundle_file_unlisted", + "bundle_symlink_forbidden", "bundle_path_unsafe", "depends_on_unresolved", + "depends_on_cycle", "seed_file_missing", "seed_file_unlisted", "process_count_exceeded", + "parallelism_out_of_range", "evidence_seam_required", "processes_required", + "processes_and_seed_forbidden", "document_only_for_compose", + "compose_runtime_requires_document", "build_command_step_empty", + "started_check_requires_exactly_one_of_port_or_log_marker", "resolved_secret_forbidden", + "capability_slug_invalid", "process_name_invalid", "fixed_port_reserved", + } +) + + +def _bundle_unavailable_code(raw_message: str) -> str: + code = raw_message.split(":", 1)[0].strip() + return code if code in _SECTION_2E_CODES else "bundle_manifest_invalid" + + +class DefaultBundleSource: + """Looks for an already-authored bundle at `work_directory / bundle_dir_name`. This is a + placeholder location this module invented (see the module-level STUCK DECISION note) — a real + bundle-authoring stage should either write there or be wired in via its own `BundleSource`.""" + + def __init__(self, bundle_dir_name: str = DEFAULT_BUNDLE_DIR_NAME) -> None: + self._bundle_dir_name = bundle_dir_name + + def load( + self, job: HarnessJob, *, source: Path, work_directory: Path + ) -> tuple[EnvironmentBundleV2, Path]: + del job, source # unused by the default (a real stage would author from these) + bundle_dir = work_directory / self._bundle_dir_name + try: + manifest = load_bundle_v2(bundle_dir) + except BundleV2Error as exc: + raise BundleUnavailableError(_bundle_unavailable_code(exc.args[0]), str(exc)) from exc + return manifest, bundle_dir + + +# ================================================================================================= +# Scenario source -- generation is Karthik's contract (in review, not available here); the +# pre-allocation CALL is this module's (ScenariosClient below). Injectable for the same reason as +# BundleSource: the glue between "generated scenarios" and "pre-allocated against the platform" can +# only be finished once that contract's payload shape lands. +# ================================================================================================= + + +class ScenarioSourceNotWired(RuntimeError): + """The default `ScenarioSource` — no Scenario Generation Contract implementation exists in this + repo yet. Raised rather than fabricating scenarios, and mapped to FAILED / `platform_sync` / + `validating_scenarios`, matching spine §5 step 3.5's own failure mapping for a pre-allocation + that never completes.""" + + +class ScenarioSource(Protocol): + async def build( + self, + job: HarnessJob, + bundle: EnvironmentBundleV2, + scenarios_client: "ScenariosClient", + *, + pool: WorldPool, + world_factory: WorldFactory, + bundle_dir: Path, + ) -> Sequence[Scenario]: ... + + +class NotWiredScenarioSource: + async def build( + self, + job: HarnessJob, + bundle: EnvironmentBundleV2, + scenarios_client: "ScenariosClient", + *, + pool: WorldPool, + world_factory: WorldFactory, + bundle_dir: Path, + ) -> Sequence[Scenario]: + del job, bundle, scenarios_client, pool, world_factory, bundle_dir + raise ScenarioSourceNotWired( + "no ScenarioSource wired -- scenario generation is not implemented in this repo yet " + "(Scenario Generation Contract, in review)" + ) + + +# ================================================================================================= +# WorldFactory -- real HostedWorld instances, fed by build.json's row counts (never +# a partial map). +# ================================================================================================= + + +class WorldFactoryError(RuntimeError): + """The provisioner handed back a runtime this factory cannot build a `World` for — a bug + upstream (no postgres endpoint despite `build.json` recording a seeded store, or `build.json` + missing the row counts for that store), never a scenario-code fault. A bundle that declared + no store at all is NOT this: that is the legal storeless lane (`StorelessWorld`, below).""" + + +def _process_runtime_error_domain(exc: ProcessRuntimeError) -> FailureDomain: + """v1.15 §2f: the producer (`process_runtime.py`) resolves and carries `domain` at the raise + site -- read it directly rather than re-deriving `spawn_failed`'s managed/source split from + the manifest (the old approach could not tell which process kind failed without one). The + imported `SECTION_2F_DOMAIN` map is a fallback ONLY, for an error that reaches here with no + carried domain -- logged when it fires, matching the scheduler's own rule. + """ + if exc.domain is not None: + return exc.domain + if exc.code in SECTION_2F_DOMAIN: + logger.warning( + "process_runtime error %r crossed the §4 seam with no carried domain; using the §2f " + "fallback map (%s)", exc.code, SECTION_2F_DOMAIN[exc.code].value, + ) + return SECTION_2F_DOMAIN[exc.code] + return FailureDomain.INFRASTRUCTURE # internal_* etc. -- the honest default + + +_SECTION_2F_CODES: frozenset[str] = frozenset(SECTION_2F_DOMAIN) + + +def _section_2f_code(code: str) -> str: + # §2f is closed (contract §4.6) -- `process_runtime.py`'s own `internal_*` codes, and this + # module's untyped-exception fallback, must never cross the outbound seam unlabeled, matching + # the discipline `_bundle_unavailable_code` already applies to §2e. The real code is + # still visible on the wire -- it stays in `message` (`ProcessRuntimeError.__str__` embeds it, + # and the untyped-exception call site prefixes it explicitly). + return code if code in _SECTION_2F_CODES else "spawn_failed" + + +def _find_postgres_endpoint(runtime: EnvironmentRuntime) -> RuntimeEndpoint | None: + for endpoint in runtime.endpoints.values(): + if endpoint.protocol == "postgres": + return endpoint + return None + + +def load_build_output(work_directory: Path) -> dict[str, Any]: + """`write_build_output` (process_runtime.py) writes `/artifacts/build.json`. + Read fresh each call — cheap, and the row counts are immutable after baseline freeze, so + re-reading is simpler than a cache invalidation story for the same modest cost.""" + path = work_directory / "artifacts" / "build.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + raise WorldFactoryError(f"build.json unreadable at {path}: {exc}") from exc + + +def row_counts_for_capability(build_output: dict[str, Any], capability: str) -> dict[str, int]: + for store in build_output.get("stores", []): + if store.get("capability") == capability: + counts = store.get("row_counts") or {} + return {str(name): int(count) for name, count in counts.items()} + raise WorldFactoryError( + f"build.json has no store entry for capability {capability!r} — the provisioner " + "guarantees a complete row-count map per store, so this bundle's build output is malformed" + ) + + +class StorelessWorld: + """The `World` handle for a bundle that declared no SQL store — a stateless agent. + + Every state-touching method raises `WorldStoreless` (a `WorldError` subclass), which the + scheduler's phase classifier routes to the typed `world_usage` receipt failure: the scenario + errors by name, the run keeps going, and nothing crashes. A scenario whose setup/ready/checks + never touch state runs and passes exactly as it would anywhere else. `read_only()` returns the + handle itself — there is nothing writable to protect, and raising there would escape the + scheduler's phase machinery instead of landing in it.""" + + def __init__(self, world_index: int, rng: random.Random) -> None: + self.world_index = world_index + self.rng = rng + + def _refuse(self, operation: str) -> WorldStoreless: + return WorldStoreless( + f"{operation}: this bundle declares no SQL store, so its worlds hold no state — " + "a scenario check that needs state cannot run against a storeless bundle" + ) + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + raise self._refuse("state()") + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + raise self._refuse("put()") + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + raise self._refuse("change()") + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + raise self._refuse("drop()") + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Any: + raise self._refuse("call()") + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + raise self._refuse("query()") + + def read_only(self) -> "StorelessWorld": + return self + + +class ProcessWorldFactory: + """Builds a real `HostedWorld` over the runtime's postgres endpoint. `AttachedPostgresStore` + (not the bare `PostgresStore`) is the correct base here — it takes a raw DSN and never manages + a container's own lifecycle, matching a hosted world where `ProcessRuntimeProvider` already + owns the postgres process. + + A runtime with no postgres endpoint forks on what `build.json` recorded: zero seeded stores + means the bundle is legitimately storeless (`StorelessWorld`); any recorded store means the + provisioner seeded one and then lost its endpoint — a bug upstream, still `WorldFactoryError`.""" + + def __init__(self, work_directory: Path) -> None: + self._work_directory = work_directory + + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> World: + endpoint = _find_postgres_endpoint(runtime) + build_output = await asyncio.to_thread(load_build_output, self._work_directory) + if endpoint is None: + if build_output.get("stores"): + raise WorldFactoryError( + f"world {runtime.world_index}: build.json records seeded stores but the " + f"runtime has no postgres-protocol endpoint in {sorted(runtime.endpoints)}" + ) + return StorelessWorld(runtime.world_index, rng) + row_counts = row_counts_for_capability(build_output, endpoint.capability) + store = AttachedPostgresStore(endpoint.address) + return await asyncio.to_thread( + HostedWorld, store, runtime.world_index, rng, row_counts + ) + + +# ================================================================================================= +# CallRunner -- the real voice track. NotWired stays the fallback for every job whose +# `agent.connector` is not `"livekit"` (absent voice config entirely, or a connector outside the +# LiveKit-dispatched voice path -- vapi/retell/auto) -- by design, not "improved" +# (hosted-execution-seams.md: `"connector": "livekit | vapi | retell | auto"`). +# ================================================================================================= + + +class CallRunnerNotWired(RuntimeError): + """Raised by `NotWiredCallRunner`. `hosted_scheduler._execute` treats any exception out of + `CallRunner.run` (other than `WorldUnavailable`/`CallAborted`) as `call_failed` + (`FailureDomain.INFRASTRUCTURE`, retried once) — so a job run with nothing wired here degrades + every scenario to one retry-then-errored receipt rather than crashing the process.""" + + +class NotWiredCallRunner: + async def run(self, scenario: Scenario, runtime: EnvironmentRuntime) -> CallOutcome: + del scenario, runtime + raise CallRunnerNotWired( + "no CallRunner wired -- the live voice-simulation call runner is a separate track" + ) + + +_LIVEKIT_CONNECTOR = "livekit" + + +def _default_build_call_runner( + adapter: "OutboundAdapter", context: CallRunnerContext +) -> CallRunner: + """The real factory: `NotWiredCallRunner` stays exactly as documented for every connector + outside the LiveKit-dispatched voice path; a `"livekit"` job gets a real `CallRunnerImpl`, + whose OWN pre-dial validation (`call_runner._check_config`) is what surfaces an + incomplete-but-present config as a typed `call_failed`/infrastructure retry -- + `capability_unavailable` stays unreachable from this seam (would require a scheduler edit; + the contract itself calls it "a follow-up, not shipped with this text").""" + if context.job.agent.connector != _LIVEKIT_CONNECTOR: + return NotWiredCallRunner() + return CallRunnerImpl(adapter, context) + + +# ================================================================================================= +# Scenario pre-allocation -- a thin client against endpoints.scenarios (outbound-channels.md v1.3 +# Authentication: bearer + X-Harness-Fence, `{"result": {...}}` envelope, job-scoped idempotent). +# Previously unowned; owned by this module now. +# ================================================================================================= + + +class ScenarioPreallocationError(RuntimeError): + def __init__(self, error: ob.ChannelError | None) -> None: + self.error = error + super().__init__("scenario pre-allocation failed" if error is None else error.message) + + +class ScenariosClient: + """RESOLVED (p13-worker-r2, reports/p13-worker-r2.md CONTRACT NOTES): Karthik's Scenario + Generation Contract (PR #63) documented two paths (`run-tests/provision/` + + `run-tests/{id}/test-executions/`) and a position-ordered `scenario_ids` response, but the + platform's actual, live route (futureagi/simulate/views/hosted_harness.py:78-90, + urls.py:128-132) mints exactly ONE url per attempt -- a DRF detail `@action` with no + `url_path`, so the router only ever produces `.../scenarios/`, never a `provision/`/`begin/` + sub-resource. The real dispatch key is a body-level `operation: "provision"|"begin"` field + (serializers/hosted_harness.py:201-226's `HarnessScenarioOperationSerializer`). This class's + transport (`_post`) is unchanged -- `provision_path`/`begin_path` are the SAME + constructor-injectable placeholders as before, now correctly defaulted to an EMPTY suffix (the + real route needs none) rather than a guessed path segment; `register_with_platform` + (scenario_source.py) is what adds the `operation` field into each payload before calling + `.provision()`/`.begin()`, matching this class's existing "operation field in payload" seam + rather than requiring a change to either method's body. Shares `channel_state` with the other + three channels (a fence on any one must stop all of them, per outbound.py's own `ChannelState` + docstring).""" + + def __init__( + self, + capabilities: ob.HostedCapabilities, + transport: ob.Transport | None = None, + *, + retry_policy: ob.RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + channel_state: ob.ChannelState | None = None, + provision_path: str = "", + begin_path: str = "", + ) -> None: + self._capabilities = capabilities + self._transport = transport or ob.RequestsTransport() + self._retry_policy = retry_policy or ob.RetryPolicy() + self._sleep = sleep + self._rng = rng + self._channel_state = channel_state or ob.ChannelState() + self._provision_path = provision_path + self._begin_path = begin_path + + def provision(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + return self._post(self._provision_path, payload, deadline=deadline) + + def begin(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + return self._post(self._begin_path, payload, deadline=deadline) + + def _post( + self, path_suffix: str, payload: dict[str, Any], *, deadline: float | None + ) -> dict[str, Any]: + self._channel_state.check() + url = f"{self._capabilities.endpoints.scenarios}{path_suffix}" + + def perform(_attempt: int) -> ob.TransportResponse: + return self._transport.request( + "POST", url, headers=self._capabilities.auth_headers(), json_body=payload + ) + + try: + response, error = ob._perform_with_retry( + perform, + retry_policy=self._retry_policy, + sleep=self._sleep, + rng=self._rng, + deadline=deadline, + ) + except (ob.HostedFencedError, ob.HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None or response is None: + raise ScenarioPreallocationError(error) + body = response.body if isinstance(response.body, dict) else {} + result = body.get("result") + if not isinstance(result, dict): + raise ScenarioPreallocationError( + ob.ChannelError( + ob.ChannelOutcome.PERMANENT_ITEM, + FailureDomain.PLATFORM_SYNC, + "scenarios_envelope_invalid", + "response body has no {'result': {...}} envelope", + ) + ) + return result + + +# ================================================================================================= +# OutboundPort adapter -- the real emit pipeline: redact -> capabilities.event_builder() -> +# spool.append -> EventsClient.flush(). Also: baseline_frozen/parallelism_degraded from build.json, +# terminal events (exactly one, last), artifact-before-receipt ordering, +# and RunResult.aborted -> TerminalFailure(infrastructure, running, "world_pool_exhausted"). +# ================================================================================================= + + +_TERMINAL_FAILURE_MESSAGE_MAX_CHARS = 4096 # an unbounded `failure.message` can blow +# EVENT_PAYLOAD_MAX_BYTES and hard-reject the WHOLE terminal event; log is the only event type +# that self-truncates. 4KB is ample for a diagnostic message. + + +def _cap_failure_message(message: str) -> str: + if len(message) <= _TERMINAL_FAILURE_MESSAGE_MAX_CHARS: + return message + marker = "…[truncated]" + return message[: _TERMINAL_FAILURE_MESSAGE_MAX_CHARS - len(marker)] + marker + + +# guest-side mirror of outbound-channels.md's artifact level table (Channel 3) -- no module +# owns this table yet (the sealer's own version lives at `artifacts.py::seal_artifacts`, scoped to +# the local-SDK path); this hosted upload path needs its own "guest enforces it first" half. +_ARTIFACT_LEVEL_FORBIDDEN_KINDS: dict[ArtifactLevel, frozenset[ob.ArtifactKind]] = { + ArtifactLevel.METADATA_ONLY: frozenset( + { + ob.ArtifactKind.RECORDING_COMBINED, ob.ArtifactKind.RECORDING_STEREO, + ob.ArtifactKind.RECORDING_CUSTOMER, ob.ArtifactKind.RECORDING_ASSISTANT, + ob.ArtifactKind.TRACE, ob.ArtifactKind.TOOL_TRACE, ob.ArtifactKind.TRANSCRIPT, + ob.ArtifactKind.OTHER, + } + ), + ArtifactLevel.TRACES: frozenset( + { + ob.ArtifactKind.RECORDING_COMBINED, ob.ArtifactKind.RECORDING_STEREO, + ob.ArtifactKind.RECORDING_CUSTOMER, ob.ArtifactKind.RECORDING_ASSISTANT, + ob.ArtifactKind.OTHER, + } + ), + ArtifactLevel.TRACES_AND_RECORDINGS: frozenset({ob.ArtifactKind.OTHER}), + ArtifactLevel.FULL: frozenset(), + # `local-only` is rejected at hosted admission (`local_only_not_hosted`) per the contract -- + # this adapter should never see it for a hosted job; forbid everything as a defensive default. + ArtifactLevel.LOCAL_ONLY: frozenset(ob.ArtifactKind), +} + + +class OutboundAdapter: + """Implements `hosted_scheduler.OutboundPort` plus the extra surface the entrypoint itself + needs (`stage_changed`, `baseline_frozen`, `parallelism_degraded`, `upload_artifact`, + `push_manifest`, `emit_terminal`) — hosted_scheduler.py only names the five methods scenario + code needs; everything else here is this module's own. + + Fencing (HostedFencedError) is caught INTERNALLY by every method, never re-raised: letting it + escape into `HostedScheduler._emit()` (which catches bare `Exception` and tries to log through + the very port that just raised) would silently swallow the fence and let the scheduler keep + working an attempt that can no longer report anything. `is_fenced` is the flag the entrypoint's + orchestration (and `cancel_requested`) polls instead. + """ + + def __init__( + self, + capabilities: ob.HostedCapabilities, + *, + events_spool: ob.OutboundSpool, + events_client: ob.EventsClient, + results_client: ob.ResultsClient, + artifacts_client: ob.ArtifactsClient, + channel_state: ob.ChannelState, + extra_secret_values: tuple[str, ...] = (), + clock: Callable[[], datetime] = lambda: datetime.now(timezone.utc), + flush_window_seconds: float = ob.FLUSH_WINDOW_SECONDS, + ) -> None: + self._capabilities = capabilities + self._spool = events_spool + self._events = events_client + self._results = results_client + self._artifacts = artifacts_client + self._channel_state = channel_state + self._extra_secret_values = extra_secret_values + self._clock = clock + # `event_builder`'s own `extra_secret_values` binding is what lets + # `build_event_record` redact `log.message`/`world_unhealthy.cause`/ + # `baseline_frozen.baseline_ref`/`terminal.failure.{code,message}` for every event this + # adapter emits -- binding it here, alongside identity, gives Channel 1 full redaction coverage. + self._event_builder = capabilities.event_builder(extra_secret_values=extra_secret_values) + self._stage_started = False + self._current_stage = HarnessStage.QUEUED + self._uploaded_digests: set[str] = set() + self._manifest_entries: list[dict[str, Any]] = [] + self._terminal_emitted = False + # §0.6 v1.14 (exit code 4): the terminal record's own spool sequence, and whether the + # platform ever permanently rejected it by name -- `terminal_undelivered` (below) needs to + # tell "this specific record landed" apart from "some flush somewhere failed." + self._terminal_sequence: int | None = None + self._terminal_rejected = False + self._scenario_counts: dict[str, int] = { + "passed": 0, "failed": 0, "errored": 0, "skipped": 0, + } + self._fenced_error: Exception | None = None + self._channel_failed_error: Exception | None = None + # the 120s flush window (§5.5) -- armed once, at whichever comes first: a cancel + # signal (`arm_flush_window` called explicitly by `run_job`'s `cancel_requested`) or the + # terminal event (`emit_terminal` below arms it itself, so no caller can forget). + self._flush_window_seconds = flush_window_seconds + self._flush_window_start: float | None = None + # job.artifacts is only known once job.json is parsed, which happens after this + # adapter is built (capabilities load, and the "no channel on a capabilities failure" + # contract, must come first) -- `configure_artifacts` below is called once it's available; + # this default is never actually exercised in practice, just a safe placeholder shape. + self._artifacts_policy = HarnessArtifactPolicy() + # `recording_headroom_bytes` stays 0 -- this adapter has no visibility into how many + # scenarios are still to run (or how large their recordings will be) at construction time, + # unlike the scheduler; sizing it here would be a guess dressed up as enforcement. + self._budget_tracker = ob.ArtifactBudgetTracker(self._artifacts_policy.max_artifact_bytes) + # `would_admit` (check) and `record` (reserve) must run as one atomic step -- two + # concurrent scenarios at W>1 racing the same remaining budget could otherwise both pass + # the check against a snapshot neither has updated yet. + self._artifact_budget_lock = asyncio.Lock() + + @property + def is_fenced(self) -> bool: + return self._fenced_error is not None + + @property + def terminal_undelivered(self) -> bool: + """The terminal was spooled (`emit_terminal` succeeded) but never confirmed delivered: the + platform permanently rejected the terminal item by name, or the spool's watermark never + reached the terminal's own sequence at all (channel exhaustion, a dead channel, or the + flush window running out before delivery). Exit 0 would claim a flush that provably never + happened. Fencing is checked by the caller first and always wins -- once fenced, whether + the terminal was ALSO undelivered is moot.""" + if self._terminal_sequence is None or self.is_fenced: + return False + return self._terminal_rejected or self._spool.watermark() < self._terminal_sequence + + @property + def scenario_counts(self) -> dict[str, int]: + return dict(self._scenario_counts) + + def configure_artifacts(self, policy: HarnessArtifactPolicy) -> None: + self._artifacts_policy = policy + self._budget_tracker = ob.ArtifactBudgetTracker(policy.max_artifact_bytes) + + def arm_flush_window(self) -> None: + if self._flush_window_start is None: + self._flush_window_start = time.monotonic() + + def deadline(self) -> float | None: + if self._flush_window_start is None: + return None + return self._flush_window_start + self._flush_window_seconds + + def _record_channel_error(self, exc: Exception) -> None: + if isinstance(exc, ob.HostedFencedError): + self._fenced_error = self._fenced_error or exc + else: + self._channel_failed_error = self._channel_failed_error or exc + logger.error("outbound channel latched: %s", exc) + + def _guarded(self, fn: Callable[[], Any]) -> Any: + """Runs one outbound client call. Catches `HostedFencedError`/`HostedChannelFailedError` + so neither escapes as an ordinary exception (see class docstring).""" + try: + self._channel_state.check() + except ( + ob.HostedFencedError, ob.HostedChannelFailedError, ob.HostedAttemptSupersededError, + ) as exc: + self._record_channel_error(exc) + return None + try: + return fn() + except (ob.HostedFencedError, ob.HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + self._record_channel_error(exc) + return None + + # -- events ------------------------------------------------------------------------------- + + def _emit_event( + self, *, stage: HarnessStage, type_: ob.OutboundEventType, payload: dict[str, Any] + ) -> None: + if self.is_fenced: + return # "stop emitting" -- no event of any type once fenced. + if self._terminal_emitted: + # `_bounded_close()` runs after the terminal is spooled -- an in-flight reconcile + # inside it can still call back into world_unhealthy/log. v1.3's "terminal ... exactly + # one, last emitted" is a hard invariant: anything after it is dropped locally, not + # spooled, rather than silently landing after the event the platform already finalized on. + # This also drops the rejected-event error log for anything the platform rejects on the + # SAME flush that carries the terminal -- diagnostic-quality only, since the rejected + # bytes are still recoverable as a `log`-kind artifact. + logger.warning( + "outbound event dropped after terminal: type=%s stage=%s", type_.value, stage.value + ) + return + event_id = f"event_{uuid.uuid4().hex}" + record = self._event_builder( + event_id=event_id, emitted_at=self._clock(), stage=stage, type=type_, payload=payload, + ) + spooled = self._spool.append(record) + if type_ is ob.OutboundEventType.TERMINAL: + self._terminal_sequence = spooled.sequence + self._current_stage = stage + + async def _aemit_event( + self, *, stage: HarnessStage, type_: ob.OutboundEventType, payload: dict[str, Any] + ) -> None: + # the spool append fsyncs the file AND its directory -- routed off the event loop so + # it never stalls every other concurrently-running scenario at W>1. + await asyncio.to_thread(self._emit_event, stage=stage, type_=type_, payload=payload) + + def stage_changed(self, to: HarnessStage) -> None: + frm = self._current_stage.value if self._stage_started else None + self._stage_started = True + self._emit_event( + stage=to, type_=ob.OutboundEventType.STAGE_CHANGED, payload={"from": frm, "to": to.value}, + ) + + def baseline_frozen(self, *, inputs_digest: str, baseline_ref: str) -> None: + self._emit_event( + stage=HarnessStage.VALIDATING_ENVIRONMENT, + type_=ob.OutboundEventType.BASELINE_FROZEN, + payload={"inputs_digest": inputs_digest, "baseline_ref": baseline_ref}, + ) + + def parallelism_degraded(self, *, requested: int, effective: int, reason: str) -> None: + self._emit_event( + stage=HarnessStage.VALIDATING_ENVIRONMENT, + type_=ob.OutboundEventType.PARALLELISM_DEGRADED, + payload={"requested": requested, "effective": effective, "reason": reason}, + ) + + def flush_events(self, *, deadline: float | None = None) -> ob.EventsFlushResult | None: + return self._guarded(lambda: self._events.flush(deadline=deadline)) + + async def aflush_events(self, *, deadline: float | None = None) -> None: + result = await asyncio.to_thread(self.flush_events, deadline=deadline) + # a rejected event's own payload never reaches the platform any other way -- surface + # it via a `log` event (error) and keep the bytes recoverable as a `log`-kind artifact, + # keyed off `dropped_records` (captured before the spool physically drops them). + if result is None or not result.rejected: + return + if self._terminal_sequence is not None and any( + entry.get("sequence") == self._terminal_sequence for entry in result.rejected + ): + # A permanent-item rejection is never retried -- the spool physically drops the record, + # so no later flush can ever redeliver it. + self._terminal_rejected = True + dropped_by_sequence = {record.sequence: record for record in result.dropped_records} + for entry in result.rejected: + sequence = entry.get("sequence") + await self.log( + level="error", + message=( + f"event sequence={sequence} rejected by the platform: " + f"{entry.get('code', 'unknown')}: {entry.get('message', '')}" + ), + ) + record = dropped_by_sequence.get(sequence) + if record is not None: + await self.upload_artifact(record.body, kind=ob.ArtifactKind.LOG) + + # -- OutboundPort (hosted_scheduler.py) ---------------------------------------------------- + + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.SCENARIO_STARTED, + payload={ + "scenario_key": scenario_key, "world_index": world_index, + "scenario_attempt": scenario_attempt, + }, + ) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.SCENARIO_RETRIED, + payload={"scenario_key": scenario_key, "from_world": from_world, "to_world": to_world}, + ) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + # world_unhealthy.cause <=200 (WorldUnhealthyPayload hard-rejects over that, so this must + # truncate BEFORE `_aemit_event`, not rely on the builder's own redaction, which runs + # after this call and could not shrink an already-too-long string back into budget). + redacted = ob.redact_outbound_text(cause, self._extra_secret_values) + if len(redacted) > 200: + redacted = redacted[:200] + await self._aemit_event( + stage=HarnessStage.RUNNING, type_=ob.OutboundEventType.WORLD_UNHEALTHY, + payload={"world_index": world_index, "cause": redacted}, + ) + + async def log(self, *, level: str, message: str) -> None: + await self._aemit_event( + stage=self._current_stage, type_=ob.OutboundEventType.LOG, + payload={"level": level, "message": message}, + ) + + async def receipt(self, receipt: ResultReceipt) -> None: + if self.is_fenced: + return + # counted only once a push is actually attempted -- counting before this point would + # include receipts that were never pushed (and the counts feed the terminal payload). + self._scenario_counts[receipt.status] = self._scenario_counts.get(receipt.status, 0) + 1 + call: dict[str, Any] | None = None + if receipt.call is not None and receipt.call.started_at is not None: + transcript_artifact = receipt.call.transcript_artifact + if transcript_artifact is not None: + bare = transcript_artifact.split(":", 1)[-1] + if bare not in self._uploaded_digests: + # null it rather than shipping a receipt the platform will 422 + # (`artifact_unknown`) wholesale -- the contract explicitly blesses a null + # `transcript_artifact` "named in a log event." + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} references un-acked transcript " + f"artifact {transcript_artifact}; nulling it" + ), + ) + transcript_artifact = None + recording_artifacts: list[str] = [] + for artifact_id in receipt.call.recording_artifacts: + bare = artifact_id.split(":", 1)[-1] if artifact_id else None + if artifact_id and bare not in self._uploaded_digests: + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} references un-acked recording " + f"artifact {artifact_id}; dropping it" + ), + ) + continue + recording_artifacts.append(artifact_id) + ended_at = receipt.call.ended_at + if ended_at is None: + # outbound.CallSummary.ended_at is a required str -- a call that started but + # never finished (CallAborted's partial) would otherwise fail build_result_receipt's + # validation and silently drop the whole receipt (HostedScheduler._emit's blanket + # except swallows it). + ended_at = receipt.call.started_at + await self.log( + level="warning", + message=( + f"receipt for {receipt.scenario_key} has no call.ended_at; substituting " + "started_at" + ), + ) + call = { + "started_at": receipt.call.started_at, + "ended_at": ended_at, + "duration_ms": receipt.call.duration_ms, + "turns": receipt.call.turns, + "transcript_artifact": transcript_artifact, + "recording_artifacts": recording_artifacts, + } + elif receipt.call is not None: + # `hosted_scheduler.CallSummary.started_at` is `str | None`, but + # `outbound.CallSummary.started_at` requires a real timestamp -- per the contract a + # call summary is only present once the call has genuinely started, so a call that + # never started is omitted here rather than shipped with a value that would fail + # `build_result_receipt`'s own validation. + await self.log( + level="warning", + message=f"receipt for {receipt.scenario_key} has no call.started_at; omitting call", + ) + failure: dict[str, Any] | None = None + if receipt.failure is not None: + # Redact before capping -- truncating first can cut a secret in half at + # the boundary and leave exact-substring redaction unable to find the surviving piece. + redacted_failure_message = ob.redact_outbound_text( + receipt.failure.message, self._extra_secret_values + ) + failure = { + "domain": receipt.failure.domain, + "stage": receipt.failure.stage, + "code": receipt.failure.code, + "message": _cap_failure_message(redacted_failure_message), + } + wire = ob.build_result_receipt( + job_id=self._capabilities.job_id, + attempt_id=self._capabilities.attempt_id, + attempt_number=self._capabilities.attempt_number, + scenario_key=receipt.scenario_key, + scenario_id=receipt.scenario_id, + scenario_attempt=receipt.scenario_attempt, + world_index=receipt.world_index, + status=receipt.status, + sub_goals=[ + {"name": g.name, "held": g.held, "reason": g.reason, "judged": g.judged} + for g in receipt.sub_goals + ], + evaluations=[_evaluation_wire(e) for e in receipt.evaluations], + call=call, + failure=failure, + extra_secret_values=self._extra_secret_values, + ) + push_result = await asyncio.to_thread(self._guarded, lambda: self._results.push(wire)) + if push_result is not None and push_result.error is not None: + # The contract's own obligation for a permanent rejection (e.g. 409 receipt_conflict, + # 422 artifact_unknown): "the platform keeps the first; guest logs, no retry." `push()` + # returns this rather than raising, so nothing inspected it before now. + await self.log( + level="error", + message=( + f"receipt for {receipt.scenario_key} rejected by the platform: " + f"{push_result.error.code}: {push_result.error.message}" + ), + ) + await self.aflush_events() + + # -- artifacts (uploaded+acked BEFORE the referencing receipt) -------------------------- + + async def upload_artifact( + self, + data: bytes, + *, + kind: ob.ArtifactKind, + scenario_key: str | None = None, + deadline: float | None = None, + ) -> str | None: + """Returns the `sha256:<64-hex>` id form the wire uses (`CallSummary.transcript_artifact`, + `ArtifactManifestEntry.artifact_id`) — never the bare hex `ArtifactsClient.upload` itself + takes, which is a different, easy-to-mix-up shape (this module's own report notes it).""" + digest = hashlib.sha256(data).hexdigest() + if digest in self._uploaded_digests: + return f"sha256:{digest}" + if self.is_fenced: + return None + # guest-side level admission + budget, both BEFORE the transport is ever touched + # ("the guest enforces it first"). + forbidden = _ARTIFACT_LEVEL_FORBIDDEN_KINDS.get(self._artifacts_policy.level, frozenset()) + if kind in forbidden: + await self.log( + level="error", + message=( + f"artifact upload refused: kind={kind.value} forbidden at " + f"level={self._artifacts_policy.level.value}" + ), + ) + return None + # check-and-reserve atomically, before the actual (slow, concurrency-safe) upload -- + # see the lock's own comment in __init__. A failed upload below leaves the reservation in + # place rather than releasing it (`ArtifactBudgetTracker` has no release primitive): a + # stuck-conservative budget is safe, an under-counted one that lets two racing uploads both + # pass admission is not. + async with self._artifact_budget_lock: + if not self._budget_tracker.would_admit(kind, len(data), digest=digest): + await self.log( + level="error", + message=( + f"artifact upload refused: budget exhausted (kind={kind.value}, " + f"size={len(data)})" + ), + ) + return None + self._budget_tracker.record(kind, len(data), digest=digest) + result = await asyncio.to_thread( + self._guarded, + lambda: self._artifacts.upload( + digest, data, kind=kind, scenario_key=scenario_key, deadline=deadline + ), + ) + if result is None or result.error is not None: + code = result.error.code if result is not None and result.error is not None else "fenced" + await self.log(level="error", message=f"artifact upload failed ({kind.value}): {code}") + return None + self._uploaded_digests.add(digest) + self._manifest_entries.append( + { + "artifact_id": f"sha256:{digest}", "kind": kind.value, "size": len(data), + "scenario_key": scenario_key, + } + ) + return f"sha256:{digest}" + + async def push_manifest(self, *, complete: bool, deadline: float | None = None) -> None: + if self.is_fenced: + return + wire = ob.build_artifact_manifest( + job_id=self._capabilities.job_id, + attempt_id=self._capabilities.attempt_id, + attempt_number=self._capabilities.attempt_number, + entries=list(self._manifest_entries), + complete=complete, + ) + await asyncio.to_thread( + self._guarded, lambda: self._artifacts.push_manifest(wire, deadline=deadline) + ) + + # -- terminal (exactly one terminal event, last emitted) -------------------------------- + + async def emit_terminal( + self, + *, + stage: HarnessStage, + reason: ob.TerminalReason | None = None, + failure: dict[str, Any] | None = None, + ) -> bool: + """Returns whether a terminal event was actually emitted (False when already emitted, or + fenced). No caller reads this return value any more -- `drain()`'s own manifest push is + gated on `is_fenced` instead; kept `bool` since a future caller may still want it.""" + if self._terminal_emitted or self.is_fenced: + return False + if failure is not None and isinstance(failure.get("message"), str): + # redact BEFORE truncating -- the inverse order can cut a secret in half at the 4KB + # boundary, and exact-substring redaction can no longer find the surviving fragment. + redacted = ob.redact_outbound_text(failure["message"], self._extra_secret_values) + failure = {**failure, "message": _cap_failure_message(redacted)} + # the latch is set AFTER a successful append (below), not before -- a raise inside + # `_emit_event` (an oversized payload, an invalid `failure.domain`) must not permanently + # disable the terminal event. + self._emit_event( + stage=stage, + type_=ob.OutboundEventType.TERMINAL, + payload={ + "stage": stage.value, + "reason": reason.value if reason is not None else None, + "failure": failure, + "scenario_counts": dict(self._scenario_counts), + }, + ) + self._terminal_emitted = True + # arm the flush window HERE, unconditionally -- "120s from the cancel signal / TTL / + # terminal event," not only when a cancel was separately observed. + self.arm_flush_window() + return True + + async def flush_terminal(self, *, deadline: float | None = None) -> bool: + """`emit_terminal` only appends the terminal record to the LOCAL spool -- a + caller that pushes something else on the wire right after (skipped receipts) would + otherwise risk `receipt()`'s own trailing `aflush_events()` delivering the terminal as a + side effect of pushing THAT receipt, landing the terminal after it on the wire. Same + bounded loop as `drain()` (a backlog bigger than one `EVENTS_MAX_BATCH` must not + strand the terminal), stopping short of the manifest push -- that still belongs after + skipped receipts, not here. Returns whether a fence was observed.""" + while True: + before = self._spool.watermark() + await self.aflush_events(deadline=deadline) + if self.is_fenced or not self._spool.pending_since_watermark(): + break + if self._spool.watermark() == before or ( + deadline is not None and time.monotonic() >= deadline + ): + break + return self.is_fenced + + async def drain(self, *, complete: bool, deadline: float | None = None) -> bool: + """Best-effort final delivery: events, then the artifact manifest (Sequencing: "terminal + event -> receipts (incl. synthesized skipped) -> manifest" -- receipts are already pushed + individually by `receipt()` as each scenario finishes). Returns whether a fence was + observed during (or before) this call -- the fence most often lands on the very flush + that carries the terminal event, so the caller's exit code must come from THIS return + value, never a fence check taken before drain() ran. + + `aflush_events` delivers at most ONE `EVENTS_MAX_BATCH`-sized batch per call -- a + backlog bigger than that (the rejected-event logging in `aflush_events` can grow one) would otherwise + strand the terminal event, the highest sequence, undelivered while still exiting 0. Loops + until the spool is actually empty, a fence is observed, a flush makes no further progress, + or the deadline is gone -- whichever comes first. + """ + while True: + before = self._spool.watermark() + await self.aflush_events(deadline=deadline) + if self.is_fenced or not self._spool.pending_since_watermark(): + break + if self._spool.watermark() == before or ( + deadline is not None and time.monotonic() >= deadline + ): + break + await self.push_manifest(complete=complete, deadline=deadline) + return self.is_fenced + + + +async def _upload_terminal_artifacts( + adapter: "OutboundAdapter", + work_directory: Path, + run_result: "RunResult | None", +) -> None: + """Channel 3's level table requires every complete manifest to carry `build`, + `result`, and `log` -- the platform refuses the manifest ack without exactly + those three, which stamped fully-completed runs as failed. At `traces` the + event trace is contract-required too, but the ack does not enforce it. + Best-effort throughout: a refusal or failure leaves the manifest listing only + what genuinely uploaded (the refusal log events raised this late are dropped + locally -- the run is already past its terminal).""" + deadline = adapter.deadline() + + build_path = work_directory / "artifacts" / "build.json" + try: + build_bytes = build_path.read_bytes() + except OSError: + build_bytes = json.dumps( + {"build_record": None, "reason": "build.json missing at finalize"} + ).encode("utf-8") + + receipts = list(run_result.receipts) if run_result is not None else [] + # Receipt text (reasons, failure fields) is redacted on every other outbound + # surface; the stored artifacts must not be the one copy that leaks a secret -- + # build.json can carry endpoint URLs with embedded credentials. + secret_values = adapter._extra_secret_values + try: + build_bytes = ob.redact_outbound_text( + build_bytes.decode("utf-8", errors="replace"), secret_values + ).encode("utf-8") + except Exception: + # Fail closed: a redaction that could not run must never let the raw + # bytes (which may carry credential-bearing endpoint URLs) reach storage. + build_bytes = json.dumps( + {"build_record": None, "reason": "build.json redaction failed at finalize"} + ).encode("utf-8") + await adapter.upload_artifact(build_bytes, kind=ob.ArtifactKind.BUILD, deadline=deadline) + + def _clean(value: str | None) -> str | None: + if value is None: + return None + return _cap_failure_message(ob.redact_outbound_text(value, secret_values)) + + result_doc = { + "schema_version": "futureagi.harness-result-set.v1", + "receipts": [ + { + "scenario_key": receipt.scenario_key, + "scenario_attempt": receipt.scenario_attempt, + "world_index": receipt.world_index, + "status": receipt.status, + "sub_goals": [ + {"name": goal.name, "held": goal.held, "judged": goal.judged, + "reason": _clean(goal.reason)} + for goal in receipt.sub_goals + ], + "failure": ( + {"code": _clean(receipt.failure.code), "stage": receipt.failure.stage, + "domain": receipt.failure.domain, + "message": _clean(receipt.failure.message)} + if receipt.failure is not None else None + ), + } + for receipt in receipts + ], + } + await adapter.upload_artifact( + json.dumps(result_doc, sort_keys=True).encode("utf-8"), + kind=ob.ArtifactKind.RESULT, deadline=deadline, ) - parser.add_argument( - "--output", required=True, type=Path, help="job artifact directory" + + log_lines = [f"scenario {r.scenario_key} attempt {r.scenario_attempt}: {r.status}" for r in receipts] + log_lines.append(f"receipts: {len(receipts)}") + await adapter.upload_artifact( + ("\n".join(log_lines) + "\n").encode("utf-8"), + kind=ob.ArtifactKind.LOG, deadline=deadline, ) + + spool_path = work_directory / EVENTS_SPOOL_DIR_NAME / "events.spool.jsonl" + try: + spool_bytes = spool_path.read_bytes() + except OSError: + spool_bytes = b"" + if spool_bytes: + await adapter.upload_artifact( + spool_bytes, kind=ob.ArtifactKind.TRACE, deadline=deadline, + ) + + +def _evaluation_wire(evaluation: Any) -> dict[str, Any]: + if evaluation.kind == "metric": + return { + # `MetricEvaluation.score: float` coerces `1` -> `1.0`; `build_result_receipt` + # digests the RAW dict before that coercion, so an int here would digest-mismatch + # against the model's own re-derivation and silently drop the receipt. + "name": evaluation.name, "kind": "metric", + "score": float(evaluation.score) if evaluation.score is not None else None, + "reason": evaluation.reason, + } + return { + "name": evaluation.name, "kind": "checkpoint", "passed": evaluation.passed, + "reason": evaluation.reason, + } + + +# ================================================================================================= +# Cancellation -- spine §0 step 7 / outbound-channels.md "Cancellation signal": the gateway writes +# `cancel_path` then sends SIGTERM; the guest stops LAUNCHING new scenarios (not killing what's +# already running) and starts the 120s flush window. +# ================================================================================================= + + +class CancelState: + def __init__(self, path: Path) -> None: + self._path = path + self._sigterm_seen = False + + def note_sigterm(self) -> None: + self._sigterm_seen = True + + def requested(self) -> bool: + return self._sigterm_seen or self._path.exists() + + def reason(self) -> ob.TerminalReason | None: + try: + raw = json.loads(self._path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return None + value = raw.get("reason") if isinstance(raw, dict) else None + try: + return ob.TerminalReason(value) + except ValueError: + return None + + +def install_sigterm_handler(cancel_state: CancelState) -> Callable[[], None]: + """Best-effort: `signal.signal` only works on the process's main thread and raises + `ValueError` anywhere else (e.g. inside a test running on a worker thread) — caught and + turned into a no-op restore, since `cancel_requested` still works off the file alone.""" + + def _handler(signum: int, frame: Any) -> None: + del signum, frame + cancel_state.note_sigterm() + + try: + previous = signal.signal(signal.SIGTERM, _handler) + except (ValueError, OSError): + return lambda: None + + def _restore() -> None: + try: + signal.signal(signal.SIGTERM, previous) + except (ValueError, OSError): + pass + + return _restore + + +def default_install_sigterm_handler(cancel_state: CancelState) -> Callable[[], None]: + return install_sigterm_handler(cancel_state) + + +# ================================================================================================= +# Dependency injection -- every seam a test needs to replace with a fake, gathered in one place so +# `run_job` itself stays pure orchestration. +# ================================================================================================= + + +@dataclass +class HostedEntrypointDeps: + load_capabilities: Callable[[], ob.HostedCapabilities] = field( + default=lambda: ob.load_capabilities() + ) + bundle_source: BundleSource = field(default_factory=DefaultBundleSource) + scenario_source: ScenarioSource = field(default_factory=NotWiredScenarioSource) + build_transport: Callable[[], ob.Transport] = field(default=lambda: ob.RequestsTransport()) + build_provider: Callable[[], WorldProvisioner] = field( + default=lambda: ProcessRuntimeProvider() + ) + # The real call runner needs `OutboundAdapter.upload_artifact` to satisfy the invariant that + # referenced artifacts are uploaded+acked BEFORE the receipt that names them -- the adapter is + # threaded in once `run_job` has built it, rather than the CallRunner reaching for a global. + # `CallRunnerContext` carries everything else `CallRunnerImpl` needs (job, bundle_dir, + # evidence_seam, the target_provider secret map, attempt_number) that the bare + # `CallRunner.run(scenario, runtime)` protocol has no room for. + build_call_runner: Callable[["OutboundAdapter", CallRunnerContext], CallRunner] = field( + default=lambda adapter, context: _default_build_call_runner(adapter, context) + ) + build_world_factory: Callable[[Path], WorldFactory] = field(default=ProcessWorldFactory) + retry_policy: Callable[[], ob.RetryPolicy] = field(default=lambda: ob.RetryPolicy()) + clock: Callable[[], datetime] = field(default=lambda: datetime.now(timezone.utc)) + cancel_path: Path = field(default_factory=lambda: Path(CANCEL_SIGNAL_PATH)) + secrets_path: Path = field(default_factory=lambda: SECRETS_PATH) + flush_window_seconds: float = ob.FLUSH_WINDOW_SECONDS + install_sigterm_handler: Callable[[CancelState], Callable[[], None]] = field( + default=default_install_sigterm_handler + ) + events_spool_dir_name: str = EVENTS_SPOOL_DIR_NAME + scenarios_client_kwargs: dict[str, Any] = field(default_factory=dict) + + def build_events_spool(self, work_directory: Path) -> ob.OutboundSpool: + return ob.OutboundSpool( + work_directory / self.events_spool_dir_name, "events", sequenced=True + ) + + def build_scenarios_client( + self, + capabilities: ob.HostedCapabilities, + transport: ob.Transport, + channel_state: ob.ChannelState, + ) -> ScenariosClient: + return ScenariosClient( + capabilities, transport, channel_state=channel_state, **self.scenarios_client_kwargs + ) + + def peek_secret_values(self) -> tuple[str, ...]: + return peek_secret_values(self.secrets_path) + + def peek_target_provider_secret_values( + self, secret_purposes: dict[str, str] + ) -> dict[str, str]: + return peek_target_provider_secret_values(self.secrets_path, secret_purposes) + + +# ================================================================================================= +# Scenario-entry validation at fetch (defense against karthik-integration-changes.md K1): the +# Scenario Generation Contract's own model may not carry `scenario_key` (or may hand back some +# other malformed shape) by the time `scenario_source.build()` returns it here, and +# `hosted_scheduler.py` reads `scenario.scenario_key`/`.sub_goals`/`.setup`/`.ready` at its own +# call sites with plain attribute access -- an attribute a pydantic/dataclass model never defined +# raises AttributeError, not a typed failure, deep inside the scheduler with no terminal event and +# a nonzero exit that reads as an infrastructure crash. Checked here with `getattr` (never direct +# attribute access) so a malformed entry is caught at the seam, before the scheduler ever touches +# it -- one bad entry fails the whole job as a typed FAILED terminal instead of crashing the guest. +# ================================================================================================= + +# No closed-vocabulary code names this defect specifically (the §2e/§2f tables are bundle/process +# concerns, not scenario-content ones) -- `scenario_preallocation_failed` is this module's own +# existing code for "the scenario set is not viable for this attempt," already scoped to stage +# `validating_scenarios`, and is reused here rather than inventing a new one. Domain `environment` +# (not `platform_sync`, its other use here): a malformed entry is a deterministic generation-stage +# content defect, not a transport failure, and fails identically on retry. +_SCENARIO_ENTRY_INVALID_CODE = "scenario_preallocation_failed" + + +def _validate_scenario_entry(entry: Any, *, index: int) -> str | None: + """Returns a human-readable defect description, or `None` if `entry` looks usable by + `hosted_scheduler.py`'s `Scenario` Protocol. Every check is a `getattr` with a default, never + a direct attribute/index access -- the whole point is to survive a shape that lacks a field + entirely, not just one that carries a wrong value. + """ + scenario_key = getattr(entry, "scenario_key", None) + if not isinstance(scenario_key, str) or not scenario_key: + return f"scenario[{index}] has no non-empty scenario_key" + label = f"scenario[{index}] ({scenario_key!r})" + if not isinstance(getattr(entry, "scenario_id", None), str): + return f"{label} has no scenario_id" + if not callable(getattr(entry, "setup", None)): + return f"{label} has no callable setup()" + if not callable(getattr(entry, "ready", None)): + return f"{label} has no callable ready()" + sub_goals = getattr(entry, "sub_goals", None) + if not isinstance(sub_goals, Sequence) or isinstance(sub_goals, (str, bytes)): + return f"{label} has no sub_goals sequence" + for goal_index, goal in enumerate(sub_goals): + goal_name = getattr(goal, "name", None) + if not isinstance(goal_name, str) or not goal_name: + return f"{label} sub_goal[{goal_index}] has no non-empty name" + if not callable(getattr(goal, "check", None)): + return f"{label} sub_goal[{goal_index}] ({goal_name!r}) has no callable check()" + return None + + +def _validate_scenarios(scenarios: Sequence[Any]) -> str | None: + for index, entry in enumerate(scenarios): + defect = _validate_scenario_entry(entry, index=index) + if defect is not None: + return defect + return None + + +# ================================================================================================= +# Orchestration -- steps 1-8, in order. +# ================================================================================================= + + +async def run_job( + job_path: Path, source: Path, output: Path, *, deps: HostedEntrypointDeps | None = None +) -> int: + """The guest's whole `main()` body. Returns the process exit code (§0.6) — `main()` below is + the only caller that turns this into `SystemExit`, so tests can call this directly and assert + on the return value.""" + deps = deps or HostedEntrypointDeps() + work_directory = output.parent + + # 1. Boot -- capabilities. CapabilitiesError -> exit non-zero-and-NOT-3, no event (v1.3 table): + # there is no channel yet to report a terminal event through. + try: + capabilities = deps.load_capabilities() + except ob.CapabilitiesError as exc: + logger.error("capabilities load failed: %s: %s", exc.code, exc.message) + return EXIT_BOOT_FAILURE + + channel_state = ob.ChannelState() + transport = deps.build_transport() + retry_policy = deps.retry_policy() + events_spool = deps.build_events_spool(work_directory) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, channel_state=channel_state + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state + ) + scenarios_client = deps.build_scenarios_client(capabilities, transport, channel_state) + + adapter = OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=deps.peek_secret_values(), + clock=deps.clock, + flush_window_seconds=deps.flush_window_seconds, + ) + + cancel_state = CancelState(deps.cancel_path) + restore_sigterm = deps.install_sigterm_handler(cancel_state) + # held outside the try so an exception on any path after this line still lets the + # `finally` below close whatever was actually provisioned. + pool: WorldPool | None = None + + def cancel_requested() -> bool: + requested = cancel_state.requested() or adapter.is_fenced + if requested: + # "120s from the cancel signal / TTL / terminal event" -- whichever comes first; + # a cancel/fence observed here starts the clock even though the terminal event (which + # also arms it, unconditionally) may not land until much later. + adapter.arm_flush_window() + return requested + + async def _bounded_close() -> None: + if pool is None: + return + remaining = adapter.deadline() + if remaining is None: + await pool.close() + return + timeout = max(0.0, remaining - time.monotonic()) + try: + await asyncio.wait_for(pool.close(), timeout=timeout) + except asyncio.TimeoutError: + logger.warning( + "pool.close() did not finish within the remaining flush window (%.1fs); " + "WorldPool already latches itself closed on entry to close(), so the top-level " + "finally's own pool.close() call cannot retry the teardown -- the provisioner may " + "be left not fully torn down until close()'s latch ordering changes", timeout, + ) + + async def _finish( + stage: HarnessStage, *, reason: ob.TerminalReason | None = None, + failure: dict[str, Any] | None = None, complete: bool, + scheduler_result: tuple[HostedScheduler, RunResult] | None = None, + ) -> int: + """Terminal event -> drain -> bounded close, in that order, for every path that + reaches a genuine terminal stage -- FAILED (via `_fail`; this now covers the pre-run + failure branches too, not just post-run ones), CANCELED, an aborted RunResult, COMPLETED. + Spending close()'s W-engine teardown time BEFORE a single terminal event is queued is + exactly the inversion this ordering guards against. + + `scheduler_result` is only ever passed by the three call sites reached AFTER + `scheduler.run()` -- pre-run terminals (`_fail`, the boundary `_canceled()` checks) have + no `RunResult` and pass nothing, so this stays a no-op there.""" + await adapter.emit_terminal(stage=stage, reason=reason, failure=failure) + # (outbound-channels.md v1.3 Sequencing): skipped receipts go out AFTER the terminal + # event, never before -- placed here so no return path below can skip this call while + # still delivering the terminal. A fenced result emits nothing further (the scheduler's + # own no-op covers it too; checked here as well so a fenced run never even attempts it). + # Best-effort like every other post-terminal emission in this module: a failure here must + # not undo the terminal already spooled above or change the exit code below. + if scheduler_result is not None: + finished_scheduler, run_result = scheduler_result + if run_result.fenced is None: + try: + # `emit_terminal` above only spools the terminal locally -- flushed to the + # wire here, BEFORE the skipped-receipt pushes below, so a receipt's own + # trailing flush can never deliver the terminal as a side effect and land it + # after that receipt on the wire. Not itself wrapped in the wait_for below -- + # it already threads the same deadline through every retry it makes, and it + # runs first, so its own delivery attempt is never the thing a timeout cuts off. + await adapter.flush_terminal(deadline=adapter.deadline()) + if not adapter.is_fenced: + # `emit_skipped_receipts`/`receipt()` have no deadline plumbing of their + # own (`push()`/`aflush_events()` run with `deadline=None`) -- a + # degraded-but-alive events channel can retry every skipped scenario's + # receipt for the full `RetryPolicy` budget, scaling with how many + # scenarios were cut short and blowing past the flush window the gateway + # tears the sandbox down at. Bounded the same way `_bounded_close` bounds + # `pool.close()`: past the deadline, stop trying and fall through to close. + remaining = adapter.deadline() + timeout = ( + None if remaining is None else max(0.0, remaining - time.monotonic()) + ) + try: + await asyncio.wait_for( + finished_scheduler.emit_skipped_receipts(run_result), + timeout=timeout, + ) + except asyncio.TimeoutError: + logger.warning( + "flush window exhausted before emit_skipped_receipts finished; " + "remaining scenarios' receipts were not sent" + ) + except Exception as exc: # noqa: BLE001 - post-terminal telemetry, never fatal + logger.error("emit_skipped_receipts failed: %s", exc) + # unlinked AFTER the terminal event, not before -- every terminal path shares the + # "secrets are no longer needed past this point" rule, but an unlink failure (a read-only + # or non-owned /run/futureagi) must never cost the one event that proves the job reached a + # terminal state at all. missing_ok=True still no-ops on paths where the provider's own + # §4.4 close() already removed the file; a genuine OSError is logged, not raised -- + # deleting an already-unneeded file is best-effort, not load-bearing. + try: + deps.secrets_path.unlink(missing_ok=True) + except OSError as exc: + logger.warning("secrets.json unlink failed: %s", exc) + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + if complete: + try: + await _upload_terminal_artifacts( + adapter, work_directory, + scheduler_result[1] if scheduler_result is not None else None, + ) + except Exception as exc: # noqa: BLE001 - post-terminal telemetry, never fatal + logger.error("terminal artifact upload failed: %s", exc) + # the exit code comes from drain()'s own post-hoc fence check (deadline computed AFTER + # emit_terminal, which is what arms the flush window), never a stale pre-drain read. + fenced = await adapter.drain(deadline=adapter.deadline(), complete=complete) + await _bounded_close() + if fenced: + return EXIT_FENCED + # §0.6 v1.14: the terminal was decided but the final drain could not flush it (the events + # channel failed) or the platform permanently rejected the terminal item itself -- exit 0 + # would claim a flush that provably never happened and silently lose the run's evidence. + if adapter.terminal_undelivered: + return EXIT_TERMINAL_UNDELIVERED + return EXIT_OK + + async def _canceled( + *, scheduler_result: tuple[HostedScheduler, RunResult] | None = None, + ) -> int: + return await _finish( + HarnessStage.CANCELED, reason=cancel_state.reason(), complete=False, + scheduler_result=scheduler_result, + ) + + async def _fail(*, domain: FailureDomain, fail_stage: HarnessStage, code: str, message: str) -> int: + # routed through `_finish` -- terminal event first, pool close (bounded) after, for + # every pre-run failure branch too, not just the post-run ones `_finish` already covered. + return await _finish( + HarnessStage.FAILED, + failure={ + "domain": domain.value, "stage": fail_stage.value, "code": code, "message": message, + }, + complete=True, + ) + + try: + # 1 (cont'd). job.json (§0.2). + try: + job = load_job(job_path) + except Exception as exc: # noqa: BLE001 - a malformed job.json has no typed error to catch + # EXIT_CRASHED (not a FAILED terminal) even though a channel now exists -- a + # malformed job.json means `job.seed`/`job.agent`/etc are not trustworthy enough to + # build a reportable failure from, and every downstream stage assumes a valid `job`. + logger.error("job.json invalid: %s", exc) + return EXIT_CRASHED + + if job.seed is None: + logger.warning("job.seed is null; spine §1 guarantees a concrete integer -- using 0") + job_seed = job.seed if job.seed is not None else 0 + parallelism = resolve_parallelism(job) + secret_purposes = job_secret_purposes(job) + # `ProcessRuntimeProvider` deletes `secrets.json` on its FIRST `provision()` call, inside + # `pool.start()` below -- this capture must happen (and does: `job` is only just now + # available, but `pool.start()` is still ~50 lines further down) strictly BEFORE that + # point, same constraint `deps.peek_secret_values()` already satisfies for redaction, + # above at adapter construction. Alias-preserving so `CallRunnerImpl` can pick e.g. + # `LIVEKIT_API_KEY` out of the map by name. + target_provider_secret_values = deps.peek_target_provider_secret_values(secret_purposes) + adapter.configure_artifacts(job.artifacts) # level table + budget, now that job.json is known. + + adapter.stage_changed(HarnessStage.VALIDATING_ENVIRONMENT) + await adapter.aflush_events() + + # Bundle authoring is not this module's (see the class docstrings above) -- injected. + try: + manifest, bundle_dir = await asyncio.to_thread( + deps.bundle_source.load, job, source=source, work_directory=work_directory + ) + except BundleUnavailableError as exc: + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_ENVIRONMENT, + code=exc.code, message=exc.message, + ) + + # 2. Preflight -- BEFORE any provision (§2e). `parallelism` is the RAW requested value + # (never clamped), so an out-of-1..8 W fails HERE with `parallelism_out_of_range`, per + # §2e.7, rather than being silently laundered into a valid one. + try: + await asyncio.to_thread( + preflight_bundle, bundle_dir, manifest, parallelism=parallelism, + secret_refs=secret_purposes, + ) + except PreflightError as exc: + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_ENVIRONMENT, + code=exc.code, message=exc.message, + ) + + # cancel/fence check at the post-preflight stage boundary. + if cancel_requested(): + return await _canceled() + + # 4/5. Provision -- ProcessRuntimeProvider, hosted lane never passes + # require_declared_user=False (the provider defaults it True on its own; the local lane's + # opt-out is a construction-site concern, not this module's). §4.5b's provider mutex is + # `WorldPool`'s own `_provider_lock` now (mutation-verified: it serializes + # provision/reset/close/healthy under one lock) -- wired directly, no extra wrapper. + provider = deps.build_provider() + pool = WorldPool( + provider, bundle=manifest, source=source, bundle_dir=bundle_dir, + work_directory=work_directory, instances=parallelism, outbound=adapter, + ) + try: + await pool.start() + except (ob.HostedFencedError, ob.HostedAttemptSupersededError): + # defensive -- nothing today routes a channel error through `pool.start()`, but a + # fenced attempt must never fall into the bare `Exception` handler below and get a + # terminal FAILED event synthesized for it. + await _bounded_close() + return EXIT_FENCED + except ob.HostedChannelFailedError as exc: + # `_fail` closes the pool itself now, AFTER the terminal event (via `_finish`) -- + # closing here first was the same close-before-terminal inversion that the terminal -> drain -> close ordering fixes elsewhere. + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + except ProcessRuntimeError as exc: + # §2f's own CARRIED domain (never the flattened `infrastructure`/"provision_failed" + # every provisioning failure used to get), stage `building_environment` per §2f. + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=_process_runtime_error_domain(exc), + fail_stage=HarnessStage.BUILDING_ENVIRONMENT, + code=_section_2f_code(exc.code), message=str(exc), + ) + except Exception as exc: # noqa: BLE001 - genuinely untyped -> infrastructure is the honest default + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.INFRASTRUCTURE, fail_stage=HarnessStage.BUILDING_ENVIRONMENT, + code=_section_2f_code("provision_failed"), message=f"provision_failed: {exc}", + ) + + # baseline_frozen + parallelism_degraded from build.json. The whole + # block is guarded -- a malformed build.json value must degrade to a `log`, never kill a + # run that has already provisioned real worlds. + degrade_emitted = False + try: + build_output = await asyncio.to_thread(load_build_output, work_directory) + except WorldFactoryError: + build_output = {} + try: + for store in build_output.get("stores", []): + if store.get("baseline_reference"): + adapter.baseline_frozen( + inputs_digest=str(store.get("inputs_digest", "")), + baseline_ref=str(store.get("baseline_reference", "")), + ) + degrade_reason = build_output.get("degrade_reason") + if degrade_reason: + requested = int(build_output.get("requested_parallelism") or parallelism) + effective = int(build_output.get("effective_parallelism") or 1) + # `ParallelismDegradedPayload` requires `1 <= effective < requested` -- + # `fixed_port` is recorded at `instances == 1` too (provider-side gap), where + # `effective == requested == 1` is not representable as a degrade at all. + if effective < requested: + adapter.parallelism_degraded( + requested=requested, effective=effective, reason=str(degrade_reason) + ) + degrade_emitted = True + else: + await adapter.log( + level="warning", + message=( + f"degrade recorded ({degrade_reason}) with requested==effective==" + f"{requested}; no parallelism_degraded event is representable" + ), + ) + except Exception as exc: # noqa: BLE001 - malformed build.json must never crash a live run + await adapter.log( + level="warning", message=f"build.json degrade/baseline block malformed: {exc}", + ) + # `pool.effective_size` is the ground truth for how many worlds actually exist -- + # if it's short of what was requested and build.json's own `degrade_reason` didn't already + # announce it (a runtime degrade build.json doesn't record), say so loudly rather + # than silently. + if not degrade_emitted and pool.effective_size < parallelism: + await adapter.log( + level="warning", + message=( + f"world pool effective_size={pool.effective_size} < requested " + f"parallelism={parallelism}, but build.json recorded no representable " + "degrade_reason" + ), + ) + await adapter.aflush_events() + + # cancel/fence check at the post-provision stage boundary. + if cancel_requested(): + return await _canceled() + + # 3. Scenario pre-allocation (spine §5 step 3.5). Generation is not this module's; the + # pre-allocation CLIENT (ScenariosClient) is. + adapter.stage_changed(HarnessStage.VALIDATING_SCENARIOS) + await adapter.aflush_events() + world_factory = deps.build_world_factory(work_directory) + # An injected `ScenarioSource` (every test, every future caller) always wins -- the real + # bundle-reading adapter (scenario_source.py) only steps in when the default + # `NotWiredScenarioSource` is still in place AND the bundle actually carries a `scenarios/` + # directory (the LAYOUT DECISION's presence test). A bundle without one keeps the existing + # typed `ScenarioSourceNotWired` failure below -- no regression for a job whose scenarios + # are not generated yet. + scenario_source = deps.scenario_source + if isinstance(scenario_source, NotWiredScenarioSource) and bundle_has_scenarios(bundle_dir): + scenario_source = BundleScenarioSource() + try: + scenarios = await scenario_source.build( + job, manifest, scenarios_client, pool=pool, world_factory=world_factory, + bundle_dir=bundle_dir, + ) + except (ob.HostedFencedError, ob.HostedAttemptSupersededError): + # `ScenariosClient._post` re-raises these after latching `channel_state` -- a fence + # here must exit 3 with no terminal event, never fall through to the generic handler. + await _bounded_close() + return EXIT_FENCED + except ob.HostedChannelFailedError as exc: + # `ScenariosClient._post` has already latched `channel_state` by the time this branch + # runs -- `emit_terminal` still spools the terminal locally (it never touches the + # network), but the drain that would flush it inherits the same latched channel and can + # never deliver. `_finish` detects exactly this (the terminal's own spool sequence never + # gets acked) and reports it honestly rather than claiming a flush that cannot happen. + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + except (ScenarioSourceNotWired, ScenarioPreallocationError) as exc: + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.PLATFORM_SYNC, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code="scenario_preallocation_failed", message=str(exc), + ) + except ScenarioDocumentInvalid as exc: + # A scenario document that will not even compile is a generation-stage content defect + # (deterministic on retry), never a transport failure -- same rationale as + # `_SCENARIO_ENTRY_INVALID_CODE`'s other use below, reused rather than inventing a new + # code for the same pair of (domain, stage). + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code=_SCENARIO_ENTRY_INVALID_CODE, message=str(exc), + ) + + # Defense against a malformed scenario entry (K1) reaching the scheduler, which reads + # `scenario_key`/`sub_goals`/`setup`/`ready` with plain attribute access and would raise + # AttributeError instead of failing the job cleanly. + scenario_defect = _validate_scenarios(scenarios) + if scenario_defect is not None: + if adapter.is_fenced: + await _bounded_close() + return EXIT_FENCED + return await _fail( + domain=FailureDomain.ENVIRONMENT, fail_stage=HarnessStage.VALIDATING_SCENARIOS, + code=_SCENARIO_ENTRY_INVALID_CODE, message=scenario_defect, + ) + + # cancel/fence check at the post-pre-allocation stage boundary. + if cancel_requested(): + return await _canceled() + + # 5/6. Scheduler wiring. + adapter.stage_changed(HarnessStage.RUNNING) + await adapter.aflush_events() + call_runner_context = CallRunnerContext( + job=job, + bundle_dir=bundle_dir, + work_directory=work_directory, + evidence_seam=manifest.runtime.evidence_seam, + target_provider_secret_values=target_provider_secret_values, + attempt_number=capabilities.attempt_number, + ) + call_runner = deps.build_call_runner(adapter, call_runner_context) + scheduler = HostedScheduler( + pool=pool, world_factory=world_factory, call_runner=call_runner, outbound=adapter, + job_seed=job_seed, cancel_requested=cancel_requested, + ) + result: RunResult = await scheduler.run(scenarios) + + # 7. Terminal + exit codes. Terminal -> drain -> close (bounded), never close() first. + if cancel_state.requested(): + return await _canceled(scheduler_result=(scheduler, result)) + if result.aborted is not None: + # v1.14 §5.4 pass-through: `result.aborted.domain`/`.code` carry straight through, not + # flattened to a fixed infrastructure/world_pool_exhausted pair -- the scheduler already + # resolves whether every world failed on the SAME never-retried §2f code (environment + # or agent domain) or a mixed set (`world_pool_exhausted`/`infrastructure`), and this + # just reports that verdict unchanged. + return await _finish( + HarnessStage.FAILED, + failure={ + "domain": result.aborted.domain, "stage": HarnessStage.RUNNING.value, + "code": result.aborted.code, "message": result.aborted.message, + }, + # An aborted run's manifest is still COMPLETE -- nothing further + # will ever upload -- and the platform accepts an incomplete + # manifest only from a canceled attempt; sending false here made + # every aborted run unackable and let the gateway overwrite the + # real abort failure with terminal_delivery_incomplete. + complete=True, + scheduler_result=(scheduler, result), + ) + # `complete: false` is reserved for the canceled lane alone. + return await _finish(HarnessStage.COMPLETED, complete=True, scheduler_result=(scheduler, result)) + finally: + if pool is not None: + try: + await pool.close() # idempotent backstop for any path above that missed one. + except Exception: # noqa: BLE001 - a finally must never mask the real exit path + logger.exception("pool.close() failed in the run_job finally backstop") + restore_sigterm() + + +# ================================================================================================= +# CLI -- spine §0 step 5's frozen invocation line: +# `python -m fi.alk.harness.hosted_entrypoint /work/job.json --source /work/source --output +# /work/artifacts` +# ================================================================================================= + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="alk-harness-worker") + parser.add_argument("job", type=Path, help="typed HarnessJob JSON (/work/job.json)") + parser.add_argument("--source", required=True, type=Path, help="/work/source checkout root") + parser.add_argument("--output", required=True, type=Path, help="/work/artifacts") args = parser.parse_args(argv) - return asyncio.run(_run(args.job, args.source, args.output)) + return asyncio.run(run_job(args.job, args.source, args.output)) if __name__ == "__main__": raise SystemExit(main()) + + +__all__ = [ + "CANCEL_SIGNAL_PATH", + "EXIT_BOOT_FAILURE", + "EXIT_CRASHED", + "EXIT_FENCED", + "EXIT_OK", + "EXIT_TERMINAL_UNDELIVERED", + "BundleSource", + "BundleUnavailableError", + "CallRunnerNotWired", + "CancelState", + "DefaultBundleSource", + "HostedEntrypointDeps", + "NotWiredCallRunner", + "NotWiredScenarioSource", + "OutboundAdapter", + "ProcessWorldFactory", + "StorelessWorld", + "ScenarioPreallocationError", + "ScenarioSource", + "ScenarioSourceNotWired", + "ScenariosClient", + "WorldFactoryError", + "install_sigterm_handler", + "job_secret_purposes", + "load_build_output", + "load_job", + "main", + "peek_secret_values", + "resolve_parallelism", + "row_counts_for_capability", + "run_job", +] diff --git a/src/fi/alk/harness/hosted_scheduler.py b/src/fi/alk/harness/hosted_scheduler.py new file mode 100644 index 00000000..c54e92e0 --- /dev/null +++ b/src/fi/alk/harness/hosted_scheduler.py @@ -0,0 +1,1816 @@ +"""World pool + scenario loop — `hosted-execution-seams.md` v1.12 §4/§5, `world-handle-interface.md` +v3.4, `outbound-channels.md` v1.3. + +Owns: leasing/releasing the W worlds `process_runtime.ProcessRuntimeProvider.provision()` hands +back, resetting a world to pristine before each scenario (spine §4.2), running one scenario's +`setup`/`ready`/checks against the world handle (the return-convention + errored-receipt table in +`world-handle-interface.md`), the fixed one-retry-on-a-fresh-world rule (spine §5 step 4), and +synthesizing a complete receipt ledger (one per scenario, `skipped` for anything never attempted). + +Decoupling, deliberate: +- `process_runtime.py` is the real, settled provisioner — imported directly (`EnvironmentRuntime`, + `RuntimeState`). `WorldProvisioner` below is a structural `Protocol` matching + `ProcessRuntimeProvider`'s actual async shape so tests can inject a fake without touching a real + filesystem/subprocess tree. +- `OutboundPort` is this module's own minimal sink for the events/receipts it produces, typed + against `outbound-channels.md`'s closed vocabulary; whoever wires the real client adapts to it. + `outbound.py` is now committed and quiescent, so this module imports exactly three of its + exception types — `HostedFencedError`/`HostedChannelFailedError`/`HostedAttemptSupersededError`, + the full `ChannelState` "stop emitting" latch for one attempt — to recognize the one outbound + failure class that is NOT best-effort (a 401/403 fence, an exhausted channel, or a superseded + attempt must stop the run, not be logged and forgotten); nothing else from that module is + imported here. +- The Scenario Generation Contract (Karthik, in review) is not available here either, so `Scenario` + is this module's own minimal Protocol for what the loop needs: a key/id pair, `setup`/`ready`, + and named sub-goal checks. Same for the simulated "call" itself (a different track's seam) — + `CallRunner` is injected. +- Secrets and the cancel signal are entrypoint-owned (P10); `cancel_requested` is an injected + zero-argument callable. +""" + +from __future__ import annotations + +import asyncio +import inspect +import logging +import random +import re +import threading +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Awaitable, Callable, Protocol, Sequence + +from .job import FailureDomain, HarnessStage +from .outbound import HostedAttemptSupersededError, HostedChannelFailedError, HostedFencedError +from .process_runtime import ( + SECTION_2F_DOMAIN, + EnvironmentRuntime, + ProcessRuntimeError, + RuntimeState, +) + +from .world.errors import ( + WorldError, + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from .world.runtime import Call + +logger = logging.getLogger(__name__) + +# --- the World handle (world-handle-interface.md v3.4) -------------------------------------- +# +# The frozen contract's code block gives six verbs plus `world_index`/`rng`. `read_only()` is not +# in that block, but the contract still requires `ready`/`check` to receive a handle whose writes +# raise `WorldReadOnly` (own section, "Read-only handles") without saying how a caller gets one — +# the shipped `HostedWorld.read_only()` (`world/handle.py`) already names this exact operation, so +# mirroring it here is the reversible choice: a real `HostedWorld` satisfies this Protocol as-is. +# +# m8: `World.read_only()` used to be typed `-> "World"`, but the real `ReadOnlyWorld` it returns +# has no `read_only()` of its own (mirroring `world/handle.py`'s own `ReadOnlyWorld`, which is +# deliberately not re-enterable) — so it fails a structural check against `World` itself. +# `ReadOnlyWorld` below names the narrower surface `ready`/`check` actually receive. + + +class ReadOnlyWorld(Protocol): + world_index: int + rng: random.Random + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: ... + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: ... + + def change( + self, collection: str, key: str, changes: dict[str, Any], *, by: str = "" + ) -> int: ... + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: ... + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: ... + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: ... + + +class World(Protocol): + world_index: int + rng: random.Random + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: ... + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: ... + + def change( + self, collection: str, key: str, changes: dict[str, Any], *, by: str = "" + ) -> int: ... + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: ... + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: ... + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: ... + + def read_only(self) -> ReadOnlyWorld: ... + + +class WorldFactory(Protocol): + """Builds the `World` handle for one already-reset `EnvironmentRuntime`. + + Deliberately not this module's job: `HostedWorld` needs a `PostgresStore` (parsed from the + runtime's `database` endpoint) plus the baseline row counts the provisioner measured at + freeze time — both live behind `ProcessRuntimeProvider`'s private state, which §4's + `RuntimeProvider` Protocol never exposes. Injected instead of guessed. + """ + + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> World: ... + + +# --- the provisioner surface this module actually drives ------------------------------------- +# +# Matches `ProcessRuntimeProvider`'s real async shape (process_runtime.py) structurally, not the +# older single-runtime `runtime.RuntimeProvider`. `bundle` stays `Any` — this module never reads a +# bundle field itself, only threads it back into `provision()` reconcile calls, so it does not +# need `EnvironmentBundleV2`'s own in-flux-adjacent type. +# +# M1 (spine v1.12 §4): `bundle_dir` is a required keyword — §2c seed/migration paths resolve +# against the verified bundle root, never against `source`. `require_declared_user` is dropped +# entirely: it is not in §4's signature, and the real provider now defaults it `True` on its own +# (the local lane opts out at provider construction, not per call). +# M2 (spine §4 point 3): `healthy` — declared readiness probes, not "process is running" — is a +# port method, not an optionally-injected callable. + + +class WorldProvisioner(Protocol): + async def provision( + self, + bundle: Any, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, + contract: Any | None = None, + instances: int = 1, + ) -> list[EnvironmentRuntime]: ... + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: ... + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: ... + + async def close(self, *, work_directory: Path) -> None: ... + + +# --- scenarios (this module's own minimal surface; Karthik's contract is not wired yet) ------- + + +class SubGoal(Protocol): + name: str + judged: str # `sub_goals[].judged` per outbound-channels.md: boolean is `judged != ""`. + + def check(self, world: ReadOnlyWorld, calls: Sequence[Call]) -> object: ... + + +class Scenario(Protocol): + scenario_key: str + scenario_id: str # platform id from pre-allocation (outbound-channels.md Channel 2 "Join"). + sub_goals: Sequence[SubGoal] + + def setup(self, world: World) -> object: ... + + def ready(self, world: ReadOnlyWorld) -> object: ... + + +# --- the simulated call (a different track's seam; injected, never built here) --------------- + + +@dataclass(frozen=True) +class CallOutcome: + calls: tuple[Call, ...] + turns: int + started_at: str | None + ended_at: str | None + duration_ms: int + transcript_artifact: str | None = None + recording_artifacts: tuple[str, ...] = () + + +class CallAborted(RuntimeError): + """The call step started but did not finish. `partial`, when known, carries whatever timing + the call runner already measured — the receipt's `call` field must not be null once the call + has genuinely started (outbound-channels.md Channel 2, "errored receipt body").""" + + def __init__(self, message: str, *, partial: CallOutcome | None = None) -> None: + super().__init__(message) + self.partial = partial + + +class CallRunner(Protocol): + async def run(self, scenario: Scenario, runtime: EnvironmentRuntime) -> CallOutcome: ... + + +# --- receipts (outbound-channels.md Channel 2; envelope fields — job_id/attempt_id/digest/etc — +# are the emitter's concern, not reproduced here) ----------------------------------------------- + + +@dataclass(frozen=True) +class SubGoalResult: + name: str + held: bool | None + reason: str | None + judged: bool + + +@dataclass(frozen=True) +class Evaluation: + name: str + kind: str # "metric" | "checkpoint" + reason: str + score: float | None = None + passed: bool | None = None + + +@dataclass(frozen=True) +class CallSummary: + started_at: str | None + ended_at: str | None + duration_ms: int + turns: int + transcript_artifact: str | None = None + recording_artifacts: tuple[str, ...] = () + + +@dataclass(frozen=True) +class ReceiptFailure: + domain: str + stage: str + code: str + message: str + + +@dataclass(frozen=True) +class ResultReceipt: + scenario_key: str + scenario_id: str + scenario_attempt: int + world_index: int | None + status: str # "passed" | "failed" | "errored" | "skipped" + sub_goals: tuple[SubGoalResult, ...] + evaluations: tuple[Evaluation, ...] + call: CallSummary | None + failure: ReceiptFailure | None + + +# --- outbound (this module's own minimal sink; see the module docstring's decoupling note) ---- + + +class OutboundPort(Protocol): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: ... + + async def scenario_retried( + self, *, scenario_key: str, from_world: int, to_world: int + ) -> None: ... + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: ... + + async def log(self, *, level: str, message: str) -> None: ... + + async def receipt(self, receipt: ResultReceipt) -> None: ... + + +# --- failure-code -> FailureDomain, and which codes retry once on a fresh world ---------------- +# +# `world_unavailable` is domain `environment` per world-handle-interface.md's own errored-receipt +# table (it overrides the table's default "domain: simulator"). `evidence_missing` is domain +# simulator but is explicitly carved out as retryable in that same document ("gets the same single +# retry-on-another-world as a world failure"). `call_failed` (v3.3) and `driver_crashed` (v3.4) are +# both rows in that same closed table now: `call_failed` is domain infrastructure, retried once +# like a world failure; `driver_crashed` is domain simulator, not retried — the scheduler's own +# machinery failing while driving a scenario, distinct from any agent/check/call outcome. +# `world_pool_exhausted` is NOT a per-scenario receipt code — it is `HostedScheduler.run()`'s own +# job-abort signal for spine v1.12 §5.4's closed job-level failure vocabulary for stage `running` +# (domain infrastructure). + +_CODE_DOMAIN: dict[str, FailureDomain] = { + "setup_crashed": FailureDomain.SIMULATOR, + "setup_timeout": FailureDomain.SIMULATOR, + "ready_timeout": FailureDomain.SIMULATOR, + "check_timeout": FailureDomain.SIMULATOR, + "ready_not_ready": FailureDomain.SIMULATOR, + "ready_broken": FailureDomain.SIMULATOR, + "check_broken": FailureDomain.SIMULATOR, + "evidence_missing": FailureDomain.SIMULATOR, + "world_usage": FailureDomain.SIMULATOR, + "world_unavailable": FailureDomain.ENVIRONMENT, + "state_too_large": FailureDomain.SIMULATOR, + "call_failed": FailureDomain.INFRASTRUCTURE, + "driver_crashed": FailureDomain.SIMULATOR, + "world_pool_exhausted": FailureDomain.INFRASTRUCTURE, +} +_RETRYABLE_CODES = frozenset({"evidence_missing"}) + +# hosted-execution-seams.md v1.13 §5.4/§2f: the closed provisioner build/run failure-code table -- +# these used to be discarded at the reset()/provision() seam (caught as a bare `Exception`, only +# `str()` surviving into `world_unhealthy.cause`), so a deterministic `environment`/`agent` fault +# (never retried) was re-reported as `world_pool_exhausted`/infrastructure and burned every +# whole-job retry on a failure that repeats identically. v1.15: the PRODUCER now resolves +# `spawn_failed`'s managed-vs-source split at the raise site (`ProcessRuntimeError.domain`) -- +# this module reads that carried domain first; `SECTION_2F_DOMAIN` (imported from +# `process_runtime.py`, the codes' own home) is consulted only for the rare error that reaches +# here with no carried domain, and that fallback is logged so a silent re-guess never hides again. + + +def _resolve_2f_domain( + code: str | None, domain: FailureDomain | None, +) -> tuple[str, FailureDomain] | None: + """v1.15 §2f: pair a code with its resolved domain. `domain` should be the value CARRIED by a + typed provisioner error (`ProcessRuntimeError.domain`); `None` here falls back to the closed + code->domain map (and logs it) rather than silently re-guessing. Returns `None` outright for + no code, or a code outside the §2f table (`internal_*` etc.) — callers already treat that the + same as "not a §2f error." + """ + if code is None or code not in SECTION_2F_DOMAIN: + return None + if domain is not None: + return code, domain + logger.warning( + "process_runtime error %r crossed the §4 seam with no carried domain; using the §2f " + "fallback map (%s)", code, SECTION_2F_DOMAIN[code].value, + ) + return code, SECTION_2F_DOMAIN[code] + + +# v1.13: only these two domains are "never retried" -- a uniform §2f code across every unhealthy +# world in one of them surfaces as that code+domain; anything else (mixed codes, or any +# infrastructure-domain fault) stays `world_pool_exhausted` exactly as before. +_SECTION_2F_NEVER_RETRIED = frozenset({FailureDomain.ENVIRONMENT, FailureDomain.AGENT}) + +# The one `OutboundPort` failure class that is NOT best-effort -- outbound-channels.md: +# 401/403 -> "stop emitting, exit code 3 ... never an infra retry," and the same latch also covers +# a 404-exhausted channel and a 409 attempt-supersession (outbound.py's `ChannelState`: "a fence in +# substance"). Letting `_emit`/`_log`/`mark_unhealthy` swallow any of these the same way they +# swallow a transport hiccup ran a superseded attempt's entire scenario set after the platform had +# already fenced or superseded it. +_FATAL_OUTBOUND: tuple[type[Exception], ...] = ( + HostedFencedError, + HostedChannelFailedError, + HostedAttemptSupersededError, +) + +# M13: an exception/overrun outcome leaves the world half-applied — world-handle-interface.md's +# return-conventions rule is "the world is discarded and re-provisioned (a half-applied world is +# never reused)." `ready_not_ready` is deliberately excluded: a precondition failing on the shared +# sealed baseline is a clean verdict, not an exception, so the world itself is still fine. +_DISCARD_ON_ERROR_CODES = frozenset( + { + "setup_crashed", + "setup_timeout", + "ready_timeout", + "check_timeout", + "ready_broken", + "check_broken", + "world_usage", + "state_too_large", + } +) + +SETUP_TIMEOUT_SECONDS = 60.0 +READY_TIMEOUT_SECONDS = 15.0 +CHECK_TIMEOUT_SECONDS = 60.0 + +_MESSAGE_LIMIT = 2000 # matches the Call.result/error truncation convention (world-handle-interface.md). +_CAUSE_LIMIT = 200 # outbound-channels.md Channel 1: `world_unhealthy.cause` free text <=200. +_USERINFO_PATTERN = re.compile(r"://[^@/]+@") + + +def _is_retryable(code: str) -> bool: + return _CODE_DOMAIN[code] in (FailureDomain.ENVIRONMENT, FailureDomain.INFRASTRUCTURE) or ( + code in _RETRYABLE_CODES + ) + + +def _truncate(text: str, limit: int = _MESSAGE_LIMIT) -> str: + if len(text) <= limit: + return text + return text[: limit - len("…[truncated]")] + "…[truncated]" + + +def _sanitize_cause(message: str) -> str: + # M7: `cause` is capped at 200 chars and must never carry endpoint credentials — postgres + # error strings routinely embed the DSN (`postgresql://user:pw@host/db`). + return _truncate(_USERINFO_PATTERN.sub("://***@", message), _CAUSE_LIMIT) + + +def _failure(code: str, message: str) -> ReceiptFailure: + return ReceiptFailure( + domain=_CODE_DOMAIN[code].value, + stage=HarnessStage.RUNNING.value, + code=code, + message=_truncate(message), + ) + + +# --- return-convention classification (world-handle-interface.md "Return conventions") -------- + + +@dataclass(frozen=True) +class _Verdict: + held: bool + reason: str | None + broken: bool + + +def _classify_ready(value: object) -> _Verdict: + if value is None or value is True: + return _Verdict(True, None, False) + if isinstance(value, str): + if value.strip() == "": + return _Verdict(True, None, False) + return _Verdict(False, value, False) + # Bare False or any other value -> broken. checks.py's `run_world_check` treats a non-None, + # non-string ready() answer the same way; a scenario hitting this cannot be told apart from a + # buggy ready.py, which is why it is `ready_broken` rather than a clean not-ready verdict. + return _Verdict(False, None, True) + + +def _classify_check(value: object) -> _Verdict: + if value is None or value is True: + return _Verdict(True, None, False) + if isinstance(value, str): + if value.strip() == "": + return _Verdict(True, None, False) + return _Verdict(False, value, False) + if value is False: + # An agent result ("the agent did something wrong"), not a broken check — matches + # checks.py's `Outcome(name, False, "False")`. + return _Verdict(False, "False", False) + return _Verdict(False, None, True) + + +# --- phase execution: budget + exception classification --------------------------------------- + + +class _PhaseTimeout(Exception): + def __init__(self, phase: str) -> None: + super().__init__(phase) + self.phase = phase + + +class _PhaseNeverStarted(Exception): + """R1: the phase's own worker thread had not even started running when its budget elapsed — + the dedicated executor was saturated, not the phase itself overrunning. Must not read as a + genuine timeout (which discards the world); the world did nothing wrong here.""" + + def __init__(self, phase: str) -> None: + super().__init__(phase) + self.phase = phase + + +class _PhaseWorldGone(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseMisuse(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseStateTooLarge(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +class _PhaseCrashed(Exception): + def __init__(self, phase: str, cause: BaseException) -> None: + super().__init__(f"{phase}: {cause}") + self.cause = cause + + +async def _invoke( + fn: Callable[..., object], + *args: object, + timeout: float, + phase: str, + executor: ThreadPoolExecutor, +) -> object: + # R1: a `threading.Event` set as the thread body's first statement — the only way to tell + # "the phase ran past its budget" (genuine overrun, world half-applied) apart from "the + # phase's thread was still queued behind others when the budget elapsed" (the scheduler's own + # executor was saturated; the world itself never touched anything). + started_flag = threading.Event() + + def _run() -> object: + started_flag.set() + return fn(*args) + + async def _call() -> object: + # B4: real scenario code (`setup`/`ready`/`check`) is synchronous, blocking psycopg calls + # — it must never run directly on the event loop, or the timeout below is purely + # decorative and every other world stalls with it. Dispatched to the scheduler's own + # dedicated executor (world-handle-interface.md: "one worker thread per world"; R1 — + # never the loop's default executor, which the provider's own `to_thread` calls also + # use). If the thread's own return value is itself awaitable (scenario code that is + # `async def`, reached indirectly through a sync wrapper), that coroutine is driven on + # the event loop afterward, where real suspension/cancellation actually works — this is + # the kept "awaitable" branch. + loop = asyncio.get_running_loop() + result = await loop.run_in_executor(executor, _run) + if inspect.isawaitable(result): + result = await result + return result + + loop = asyncio.get_running_loop() + started = loop.time() + try: + return await asyncio.wait_for(_call(), timeout=timeout) + except asyncio.TimeoutError as exc: + # m4: `asyncio.TimeoutError is TimeoutError` on 3.11 — a psycopg statement timeout raised + # INSIDE `fn` looks identical to `wait_for`'s own deadline unless the elapsed time is + # actually checked. If the budget did not genuinely elapse, this was `fn`'s own timeout + # bubbling through — a broken phase, not a budget overrun. + if loop.time() - started < timeout: + raise _PhaseCrashed(phase, exc) from exc + if not started_flag.is_set(): + raise _PhaseNeverStarted(phase) from exc + # B4: `wait_for`'s cancellation stops US from waiting on the thread, not the thread + # itself — psycopg in-flight cancellation is not wired here (P11 follow-up; recorded in + # the fixer report). The thread is abandoned, bounded by scenario count per the contract's + # own accepted tradeoff; its world is discarded rather than reused (M13). + raise _PhaseTimeout(phase) from exc + except WorldUnavailable as exc: + raise _PhaseWorldGone(phase, exc) from exc + except WorldStateTooLarge as exc: + raise _PhaseStateTooLarge(phase, exc) from exc + except (WorldReadOnly, WorldReservedName, WorldQueryRejected, WorldUsageError) as exc: + raise _PhaseMisuse(phase, exc) from exc + except WorldError as exc: + # m5: catches any WorldError subclass not special-cased above (world/errors.py's own + # base, kept exactly for "route 'scenario code misused the handle' to one outcome without + # naming all six") — a future seventh subclass lands here instead of silently falling into + # the generic crash classification below. + raise _PhaseMisuse(phase, exc) from exc + except Exception as exc: + raise _PhaseCrashed(phase, exc) from exc + + +_CRASH_CODE_BY_PHASE = {"setup": "setup_crashed", "ready": "ready_broken", "check": "check_broken"} +_TIMEOUT_CODE_BY_PHASE = {"setup": "setup_timeout", "ready": "ready_timeout", "check": "check_timeout"} + + +@dataclass(frozen=True) +class _PhaseResult: + value: object + failure: ReceiptFailure | None + + +async def _run_phase( + fn: Callable[..., object], + *args: object, + timeout: float, + phase: str, + executor: ThreadPoolExecutor, +) -> _PhaseResult: + try: + value = await _invoke(fn, *args, timeout=timeout, phase=phase, executor=executor) + return _PhaseResult(value, None) + except _PhaseNeverStarted: + # R1: not the phase's fault and not the world's — the scheduler's own thread pool + # couldn't service it in time. `driver_crashed` is not in `_DISCARD_ON_ERROR_CODES`, so + # this releases the world rather than discarding a perfectly healthy one. + return _PhaseResult( + None, + _failure( + "driver_crashed", f"{phase} never started before its budget elapsed (thread pool saturated)" + ), + ) + except _PhaseTimeout: + return _PhaseResult(None, _failure(_TIMEOUT_CODE_BY_PHASE[phase], f"{phase} exceeded its budget")) + except _PhaseWorldGone as exc: + return _PhaseResult(None, _failure("world_unavailable", str(exc.cause))) + except _PhaseMisuse as exc: + return _PhaseResult(None, _failure("world_usage", str(exc.cause))) + except _PhaseStateTooLarge as exc: + return _PhaseResult(None, _failure("state_too_large", str(exc.cause))) + except _PhaseCrashed as exc: + return _PhaseResult( + None, _failure(_CRASH_CODE_BY_PHASE[phase], f"{type(exc.cause).__name__}: {exc.cause}") + ) + + +# --- the world pool ----------------------------------------------------------------------------- + + +class NoWorldsAvailable(RuntimeError): + """Every provisioned world is down and none is currently recoverable (spine v1.12 §5.4: + "if ready worlds reach 0 the job FAILS in stage running, domain infrastructure" — declared + only after in-flight re-provisioning completes without restoring a world), OR the pool has + been closed (R5: `reason="closed"`). + + v1.13 §5.4: `code`/`domain` carry a uniform §2f never-retried code when every unhealthy + world's last re-provision attempt agreed on one — `None` (the default) means the caller falls + back to the generic `world_pool_exhausted`/infrastructure abort, exactly as before.""" + + def __init__( + self, + message: str, + *, + reason: str = "exhausted", + code: str | None = None, + domain: FailureDomain | None = None, + ) -> None: + super().__init__(message) + self.reason = reason + self.code = code + self.domain = domain + + +_RECONCILE_MAX_ATTEMPTS = 3 +_RECONCILE_BACKOFF_SECONDS = (0.05, 0.1) +_LEASE_POLL_INTERVAL_SECONDS = 0.02 + + +class WorldPool: + """Leases/releases the W worlds `provisioner.provision()` returns, resets one to pristine on + every lease (spine §4.2), and reconciles an unhealthy world back in via `provision()` again + (§4 rule 1: "a sick world mid-job is recovered by calling `provision` again") — in the + background, so a lease elsewhere never blocks on someone else's recovery. + + B1/B2/M6 (spine v1.12 §4.5b): the provider port is NOT reentrant — at most one + `provision`/`reset`/`healthy`/`close` call is ever in flight, serialized by `_provider_lock` + (R13: `healthy` writes — it demotes state — so v1.12 folded it into the same serialized set + that provision/reset/close were already in; it is no longer treated as a read-only probe + exempt from the lock). A demotion that lands while a reconcile is already running is coalesced + into a trailing pass rather than a second concurrent `provision()` call. + """ + + def __init__( + self, + provisioner: WorldProvisioner, + *, + bundle: Any, + source: Path, + bundle_dir: Path, + work_directory: Path, + instances: int, + outbound: OutboundPort | None = None, + ) -> None: + self._provisioner = provisioner + self._bundle = bundle + self._source = source + self._bundle_dir = bundle_dir + self._work_directory = work_directory + self._instances = instances + self._outbound = outbound + + self._runtimes: dict[int, EnvironmentRuntime] = {} + self._available: set[int] = set() + self._leased: set[int] = set() + self._down: set[int] = set() + self._fresh: set[int] = set() # m9: provisioned/recovered but never yet leased/reset + self._effective_size = 0 # R2: the achieved world count `start()` settled on + # The §2f (code, domain) pair (or `None`) behind the most recent demotion/reconcile-failure + # for a down world index -- read by `lease()`'s exhaustion check to decide whether a + # uniform never-retried code can surface instead of the generic `world_pool_exhausted`. + # `domain` is the CARRIED value off the typed error (v1.15), captured once here rather than + # re-derived later from the code alone. + self._down_codes: dict[int, tuple[str, FailureDomain] | None] = {} + self._fenced: BaseException | None = None # latched by mark_fenced(), never cleared + + # m1: `asyncio.Condition` (not a manual `Event` + `clear()`) — waiting and notifying share + # one lock, so there is no window between releasing a lock and clearing a flag for a + # `set()` to land in and be silently lost. + self._state_lock = asyncio.Condition() + self._provider_lock = asyncio.Lock() + self._reconcile_task: asyncio.Task[None] | None = None + self._reconcile_pending = False + self._started = False + self._closing = False # R4: set at the top of close() -- lets an in-flight reconcile bail + # between attempts instead of burning close()'s wait budget on a pool being torn down. + self._closed = False # Set once close() STARTS -- latches provision()/lease() out for + # good immediately, independent of whether teardown itself has finished. + self._teardown_task: asyncio.Task[None] | None = None # the shared, retry-safe + # teardown -- see close()'s own comment for why idempotency lives here now, not on + # `_closed`. + + @property + def effective_size(self) -> int: + """R2: the world count `start()` actually achieved — may be less than the requested + `instances` on a legitimate degrade (conformance-gate failure, `fixed_port`). P10 sizes + `parallelism_degraded` and anything else that needs "how many worlds do we really have" + off this, never off the originally requested `instances`.""" + return self._effective_size + + @property + def fenced(self) -> BaseException | None: + """The first fatal `OutboundPort` exception (401/403 -> `HostedFencedError`, a + 404-exhausted channel -> `HostedChannelFailedError`, or a 409 attempt-supersession -> + `HostedAttemptSupersededError`) observed anywhere along this pool's own emit paths. + `HostedScheduler` polls this at the same points it polls `cancel_requested` to stop + leasing/launching further scenarios once set.""" + return self._fenced + + def mark_fenced(self, exc: BaseException) -> None: + if self._fenced is None: + self._fenced = exc + + @property + def size(self) -> int: + return len(self._runtimes) + + async def start(self) -> list[EnvironmentRuntime]: + if self._started: + # m10: a second call would re-provision behind every already-leased world's back. + raise RuntimeError("WorldPool.start() called more than once") + self._started = True + + async with self._provider_lock: + runtimes = await self._provisioner.provision( + self._bundle, + source=self._source, + bundle_dir=self._bundle_dir, + work_directory=self._work_directory, + instances=self._instances, + ) + + # R2 (spine v1.12 §4's conformance gate / `fixed_port`): `provision()` legitimately + # returns FEWER than `instances` worlds — "Fail → effective parallelism 1 + + # parallelism_degraded ... Loud, never silent," not a failure this pool should raise on. + # Reject only a genuinely malformed result: zero worlds, duplicates, a non-contiguous + # index set, or more worlds than were ever requested. + indices = {runtime.world_index for runtime in runtimes} + if ( + not runtimes + or len(runtimes) != len(indices) + or indices != set(range(len(runtimes))) + or len(runtimes) > self._instances + ): + # m10/R2: spine §4 — "ordered by world_index" and contiguous from 0 (what + # `range(effective_instances)` on the provider side guarantees). + raise RuntimeError( + f"provision() returned world_index set {sorted(indices)}, expected a contiguous " + f"0..N-1 subset of 0..{self._instances - 1}" + ) + self._effective_size = len(runtimes) + + async with self._state_lock: + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + if runtime.state in (RuntimeState.READY, RuntimeState.PREPARING): + # m10: never hand out a world provision() itself returned UNHEALTHY. A + # PREPARING world legitimately demotes straight to UNHEALTHY on a failed first + # reset/probe (spine v1.12 §3's preparing->unhealthy transition) -- lease()'s + # own health gate covers that case; nothing extra is needed here. + self._available.add(runtime.world_index) + if runtime.state is RuntimeState.READY: + self._fresh.add(runtime.world_index) + else: + self._down.add(runtime.world_index) + self._state_lock.notify_all() + return runtimes + + def _reconcile_in_flight(self) -> bool: + return self._reconcile_task is not None and not self._reconcile_task.done() + + async def _wait_bounded(self, *, poll: bool) -> None: + if not poll: + await self._state_lock.wait() + return + try: + await asyncio.wait_for(self._state_lock.wait(), timeout=_LEASE_POLL_INTERVAL_SECONDS) + except asyncio.TimeoutError: + pass # `Condition.wait()` reacquires the lock before propagating even on timeout. + + async def lease( + self, *, exclude: frozenset[int] = frozenset(), abandon: Callable[[], bool] | None = None + ) -> tuple[int, EnvironmentRuntime] | None: + """Returns `None` if `abandon()` reports true while this call was queued (B5) — the caller + never received a world, so there is nothing to release.""" + while True: + # R5: latched once close() has run — a lease past that point must never spawn a + # `reset()`/`healthy()` call against a provider that may already be hard-cleaned. + if self._closed: + raise NoWorldsAvailable("world pool is closed", reason="closed") + if abandon is not None and abandon(): + return None + + async with self._state_lock: + candidates = self._available - exclude + if candidates: + world_index = min(candidates) + self._available.discard(world_index) + self._leased.add(world_index) + skip_reset = world_index in self._fresh # m9 + self._fresh.discard(world_index) + else: + # Not just "every world is down" (the plain retry-exhausted case) — a world + # excluded for this lease (a same-scenario retry avoiding its failed world) + # can never satisfy `candidates` again no matter how long we wait, so it must + # count as unusable here too or a single-world pool's retry blocks forever. + usable = set(self._runtimes) - exclude + if not (usable - self._down): + # M9/R10 (spine v1.12 §5.4): declare exhaustion only once no reconcile is + # in flight or about to be — never on an instantaneous snapshot of world + # states. `_reconcile_pending` (set inside `mark_unhealthy`'s own critical + # section, R10) covers the gap between a demotion and its reconcile task + # actually existing. + if self._reconcile_in_flight() or self._reconcile_pending: + await self._wait_bounded(poll=abandon is not None) + continue + # v1.13 §5.4: a uniform §2f never-retried code across every + # currently-unhealthy world surfaces AS that code+domain; mixed codes, an + # unrecorded (non-§2f) cause, or any infrastructure-domain code all fall + # back to the generic `world_pool_exhausted` exactly as before. The + # uniformity set is `self._down` -- every unhealthy world in the pool, not + # just `usable` (runtimes minus this call's `exclude`) -- a world excluded + # because it is the scenario's own just-failed world is still part of "every + # unhealthy world" the spec means; narrowing to `usable` would let that + # excluded world's own (possibly untyped) failure escape the check entirely. + codes = {self._down_codes.get(index) for index in self._down} + code = domain = None + if len(codes) == 1: + (only,) = codes + if only is not None: + only_code, only_domain = only + if only_domain in _SECTION_2F_NEVER_RETRIED: + code, domain = only_code, only_domain + raise NoWorldsAvailable( + f"{len(self._down)}/{len(self._runtimes)} worlds unhealthy, " + f"none available outside {sorted(exclude)}", + code=code, + domain=domain, + ) + await self._wait_bounded(poll=abandon is not None) + continue + + reset_exc: Exception | None = None + probed_runtime: EnvironmentRuntime | None = None + if not skip_reset: + async with self._provider_lock: + if self._closed: + # close() can win the `_provider_lock` FIFO queue against a lease already + # past the top-of-loop `_closed` check -- re-check on the inside too, or + # this lease drives reset() against a provider close() may already be + # hard-cleaning. + raise NoWorldsAvailable("world pool is closed", reason="closed") + runtime = self._runtimes.get(world_index) + probed_runtime = runtime + if runtime is not None: + try: + await self._provisioner.reset(runtime, work_directory=self._work_directory) + except Exception as exc: # noqa: BLE001 - B3: must never leak out of lease() + reset_exc = exc + + is_healthy = False + if reset_exc is None: + # M2: `healthy()` is called unconditionally after reset — including the m9 fast + # path, which skips only the (expensive) reset call, never the readiness check. + # R13 (spine v1.12 §4.5b): `healthy` now rides the port's non-reentrancy rule too, + # so it goes under `_provider_lock` like reset/provision/close. + async with self._provider_lock: + if self._closed: + raise NoWorldsAvailable("world pool is closed", reason="closed") # same re-check as above + runtime = self._runtimes.get(world_index) + probed_runtime = runtime + if runtime is not None: + try: + is_healthy = await self._provisioner.healthy( + runtime, work_directory=self._work_directory + ) + except Exception as exc: # noqa: BLE001 + reset_exc = exc + + async with self._state_lock: + # m2/R14: re-read after the awaited provider calls — a concurrent reconcile may + # have replaced or dropped this index's `EnvironmentRuntime` while lease() awaited. + # `is_healthy` was computed against `probed_runtime` specifically; if the object + # at this index is no longer that same object, the verdict no longer describes it + # — discard this attempt and let the outer loop re-evaluate the index fresh rather + # than apply a stale verdict to a new object. + runtime = self._runtimes.get(world_index) + if runtime is None or runtime is not probed_runtime: + self._leased.discard(world_index) + if runtime is not None: + # The object was REPLACED, not removed -- put the index back where the + # outer loop can find it, or it lands nowhere (not available, not down) + # and every future lease() spins forever on a candidate set that never + # grows. + self._available.add(world_index) + self._state_lock.notify_all() + continue + if is_healthy and runtime.state is RuntimeState.READY: + self._state_lock.notify_all() + return world_index, runtime + cause = ( + f"reset failed: {reset_exc}" + if reset_exc is not None + else f"reset left world in state {runtime.state.value}" + ) + # Preserve a typed §2f code (and its CARRIED domain, v1.15) across this seam + # instead of flattening it to free text -- `mark_unhealthy` records it so a later + # exhaustion declaration can tell a deterministic never-retried fault apart from a + # generic infrastructure one. + is_typed = isinstance(reset_exc, ProcessRuntimeError) + code = reset_exc.code if is_typed else None + domain = reset_exc.domain if is_typed else None + + await self.mark_unhealthy(world_index, cause=cause, code=code, domain=domain) + # loop again — this index is now excluded via `_down`, no explicit retry bookkeeping. + + async def release(self, world_index: int) -> None: + async with self._state_lock: + self._leased.discard(world_index) + if world_index in self._runtimes and world_index not in self._down: + self._available.add(world_index) + self._state_lock.notify_all() + + async def mark_unhealthy( + self, + world_index: int, + *, + cause: str, + code: str | None = None, + domain: FailureDomain | None = None, + ) -> None: + # `domain` is the CARRIED value off a typed provisioner error (v1.15); `None` here (e.g. a + # caller that only has a bare code) falls back to the closed map via `_resolve_2f_domain`, + # logged when it fires. + async with self._state_lock: + self._leased.discard(world_index) + self._available.discard(world_index) + self._fresh.discard(world_index) + self._down.add(world_index) + # Unconditional -- every demotion overwrites the recorded reason (or clears a stale + # §2f code with `None` when this one isn't typed), so exhaustion always reads the + # MOST RECENT cause for this index, never a leftover from an earlier failure. + self._down_codes[world_index] = _resolve_2f_domain(code, domain) + runtime = self._runtimes.get(world_index) + if runtime is not None: + # M12 (spine v1.12 §4.5b, normative): the scheduler demotes `state` on the + # provider's own live `EnvironmentRuntime` object — that demotion is the signal + # the NEXT `provision()` reconciles on. + runtime.state = RuntimeState.UNHEALTHY + # R10: set inside this same critical section (not left to `_schedule_reconcile`'s own, + # later one) so a `lease()` observing state in the gap between the two never sees + # "every world down, no reconcile in flight or pending" and raises spuriously. + self._reconcile_pending = True + self._state_lock.notify_all() + + # Schedule recovery BEFORE the telemetry emit below -- `OutboundPort` calls are + # best-effort and may be slow or hang, and recovery must never sit behind one (worst + # case: `_reconcile_pending` stays latched and `lease()`'s grace loop spins forever). + await self._schedule_reconcile() + + # R6: this is the sole path every demotion (this method) goes through, so it is the one + # place `world_unhealthy` needs to be emitted from for all four call sites to get it. + if self._outbound is not None: + try: + await self._outbound.world_unhealthy(world_index=world_index, cause=_sanitize_cause(cause)) + except _FATAL_OUTBOUND as exc: # a fence stops the run -- never best-effort. + self.mark_fenced(exc) + except Exception as exc: # noqa: BLE001 - B3: outbound failures are never fatal. + await self._log(f"world_unhealthy emit failed: {exc}") + + async def _schedule_reconcile(self) -> None: + async with self._state_lock: + if self._closed: + return # R5: never spawn new provider work once the pool has been closed. + if self._reconcile_in_flight(): + # B1/M6: a demotion landing mid-reconcile is coalesced into a trailing pass + # (`_reconcile_loop`) rather than a second concurrent `provision()` call. + self._reconcile_pending = True + return + self._reconcile_task = asyncio.create_task(self._reconcile_loop()) + + async def _reconcile_loop(self) -> None: + while True: + async with self._state_lock: + self._reconcile_pending = False + await self._reconcile() + async with self._state_lock: + if not self._reconcile_pending: + return + + async def _reconcile(self) -> None: + # M5: bounded retry with backoff — a single transient `provision()` failure (a momentary + # ENOSPC, an engine hiccup) used to retire its world for the rest of the job with no + # signal anywhere. Every failed attempt is logged through `OutboundPort` (when wired), + # matching the contract's own "loud, never silent" standard for degradation. + runtimes: list[EnvironmentRuntime] | None = None + last_exc: Exception | None = None + for attempt in range(1, _RECONCILE_MAX_ATTEMPTS + 1): + if self._closing: + # R4: close() is already bounded-waiting on this task — do not spend its wait + # budget retrying a pool that is being torn down anyway. + return + try: + async with self._provider_lock: + runtimes = await self._provisioner.provision( + self._bundle, + source=self._source, + bundle_dir=self._bundle_dir, + work_directory=self._work_directory, + instances=self._instances, + ) + except Exception as exc: # noqa: BLE001 - a reconcile must never crash the pool + last_exc = exc + await self._log( + f"world pool reconcile attempt {attempt}/{_RECONCILE_MAX_ATTEMPTS} failed: {exc}" + ) + if attempt < _RECONCILE_MAX_ATTEMPTS and not self._closing: + await asyncio.sleep(_RECONCILE_BACKOFF_SECONDS[attempt - 1]) + continue + last_exc = None + break + + if last_exc is not None or runtimes is None: + # The FINAL failed re-provision attempt's typed §2f code (and its CARRIED domain, + # v1.15), applied to every world still down when this reconcile gives up -- one + # `provision()` call covers the whole pool, so a typed failure here is uniform by + # construction across everything it did not just recover. + is_typed = isinstance(last_exc, ProcessRuntimeError) + code_and_domain = _resolve_2f_domain( + last_exc.code if is_typed else None, last_exc.domain if is_typed else None, + ) + # R8: every success path below ends in `notify_all()` — this give-up path must too, + # or a `lease()` blocked in `_wait_bounded(poll=False)` (the `abandon is None` case) + # waits forever for a reconcile that already gave up. + # Unconditional, mirroring `mark_unhealthy`'s own invariant -- an untyped final + # attempt must overwrite (clear) a stale typed code left by an earlier demotion, or + # exhaustion later reads that leftover code as if it were this attempt's own result. + async with self._state_lock: + for index in self._down: + self._down_codes[index] = code_and_domain + self._state_lock.notify_all() + return # stays `_down`; the next `mark_unhealthy` (or a lease-triggered wait) retries. + + # M12: recovery is judged by re-probing `healthy()` (M2's port), never by reading `state` + # back — the scheduler is what wrote `state` when it demoted this world, so trusting it + # here would be reading our own signal as independent proof. R13 (spine v1.12 §4.5b): + # `healthy` now rides the port's non-reentrancy rule, so these probes go under + # `_provider_lock` too. + if self._closing: + # provision() just succeeded, but close() may already be queued on `_provider_lock` + # for its own `provisioner.close()` call -- bail before racing it for one more round + # of provider calls the pool is being torn down under anyway. + return + healthy_by_index: dict[int, bool] = {} + # The probe's own §2f code, carried alongside its verdict -- a world that comes back from a + # SUCCESSFUL `provision()` but fails this probe never enters the give-up path above (that + # path only fires on a raised/failed `provision()`), so without this the state block below + # has no code of its own and would otherwise leave whatever an earlier, superseded demotion + # recorded standing. + healthy_codes: dict[int, tuple[str, FailureDomain] | None] = {} + async with self._provider_lock: + for runtime in runtimes: + try: + healthy_by_index[runtime.world_index] = await self._provisioner.healthy( + runtime, work_directory=self._work_directory + ) + healthy_codes[runtime.world_index] = None + except Exception as exc: # noqa: BLE001 + healthy_by_index[runtime.world_index] = False + is_typed = isinstance(exc, ProcessRuntimeError) + healthy_codes[runtime.world_index] = _resolve_2f_domain( + exc.code if is_typed else None, exc.domain if is_typed else None, + ) + + achieved = {runtime.world_index for runtime in runtimes} + async with self._state_lock: + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + if healthy_by_index.get(runtime.world_index, False): + was_down = runtime.world_index in self._down + self._down.discard(runtime.world_index) + self._down_codes.pop(runtime.world_index, None) # recovered -- stale now + if runtime.world_index not in self._leased: + self._available.add(runtime.world_index) + if was_down and runtime.state is RuntimeState.READY: + self._fresh.add(runtime.world_index) # m9 + elif runtime.world_index in self._down: + # Still down after a successful re-provision -- this probe's own result + # replaces whatever an earlier demotion left, never a leftover from before it. + self._down_codes[runtime.world_index] = healthy_codes.get(runtime.world_index) + # `provision` reconciles to exactly `instances` worlds (a conformance-gate degrade can + # shrink `achieved` below what this pool started with) — anything no longer returned + # is gone, not merely unhealthy. + for stale in [index for index in self._runtimes if index not in achieved]: + self._runtimes.pop(stale, None) + self._available.discard(stale) + self._down.discard(stale) + self._fresh.discard(stale) + self._down_codes.pop(stale, None) # the index itself is gone + # m3: NOT `_leased.discard(stale)` — an in-flight scenario may still hold this + # index's lease (e.g. a conformance degrade shrinking `achieved` mid-scenario); + # dropping the lease record here would make its later `release()`/ + # `mark_unhealthy()` a silent no-op. Those methods already guard on + # `world_index in self._runtimes`, so leaving `_leased` alone and letting them + # reconcile it lazily is correct. + # Keep this truthful across a reconcile, not just at start() -- P10 sizes + # `parallelism_degraded` off it, and a reconcile can grow the pool back up or shrink + # it further (a conformance degrade narrowing `achieved`) in either direction. + self._effective_size = len(self._runtimes) + self._state_lock.notify_all() + + async def close(self) -> None: + async with self._state_lock: + if not self._closed: + self._closed = True + self._closing = True + # Wake anything blocked in `lease()`'s `_wait_bounded(poll=False)` so it + # re-checks `_closed` instead of waiting for a recovery that will never come. + self._state_lock.notify_all() + # The OLD idempotency check (`if self._closed: return`) latched here, before teardown + # ever ran -- a caller wrapping this whole call in its own timeout (the entrypoint's + # `_bounded_close`) could cancel it mid-teardown, and a RETRY then hit that early + # return and silently never called `provisioner.close()` at all. `_closed` still has + # to latch immediately (lease()'s top-of-loop check, its inner re-check under + # `_provider_lock`, and `_teardown`'s own bail-out below all depend on new + # leases/reconciles being rejected the moment close() STARTS, not once it finishes), + # so idempotency now lives on a separate, SHARED teardown task instead: every call — + # first or retried — creates it once and awaits the same one. + if self._teardown_task is None: + self._teardown_task = asyncio.create_task(self._teardown()) + teardown_task = self._teardown_task + + # `asyncio.shield`: if THIS call's own awaiter is cancelled (the caller's timeout fires), + # the cancellation stops at this `await` and never reaches `teardown_task` -- teardown + # keeps running in the background, and a retry's `close()` re-attaches to the same + # (possibly by-then-finished) task instead of no-op'ing. + await asyncio.shield(teardown_task) + + async def _teardown(self) -> None: + async with self._state_lock: + task = self._reconcile_task + if task is not None: + # §4.5b: do NOT cancel-then-close. + # `ProcessRuntimeProvider.provision`/`reset`/`healthy` are `asyncio.to_thread` — + # cancelling the awaiting coroutine does NOT stop the underlying thread, so the old + # bounded-wait-then-cancel let `provisioner.close()` run CONCURRENTLY with a + # still-live `provision()` once the bound expired: unsynchronized identity dicts + # (`RuntimeError: dictionary changed size during iteration`), leaked engines, and + # `close()` itself could raise out of the guest's terminal path. `_closing` (in + # `_reconcile`'s own retry loop and healthy-probe gate) already makes a reconcile bail + # BETWEEN attempts/probes without a cancel, so this waits for the ONE call already in + # flight to finish on its own — unbounded from this function's perspective, but + # bounded in practice by whichever single `provision()`/`healthy()` call was running, + # with the outer flush-window deadline (spine, P10-owned) as the real backstop -- + # there is no longer a single constant here that bounds this wait on its own. + await asyncio.gather(task, return_exceptions=True) + + async with self._provider_lock: + await self._provisioner.close(work_directory=self._work_directory) + + async def _log(self, message: str, *, level: str = "error") -> None: + if self._outbound is None: + return + try: + # R9: reuses `world_unhealthy.cause`'s own sanitizer — a `provision()` failure + # routinely carries a postgres error string with the DSN, and outbound-channels.md + # requires redaction (no endpoint userinfo) before anything crosses the wire. + await self._outbound.log(level=level, message=_sanitize_cause(message)) + except _FATAL_OUTBOUND as exc: # a fence stops the run -- never best-effort. + self.mark_fenced(exc) + except Exception: # noqa: BLE001 - B3: outbound failures are never fatal. + pass + + +# --- the scenario loop -------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class RunResult: + """`receipts` mixes already-emitted real receipts with synthesized-but-not-yet-emitted + `skipped` ones (R6) — see `HostedScheduler.emit_skipped_receipts`. + + `fenced` is set once a 401/403 (`HostedFencedError`), a 404-exhausted channel + (`HostedChannelFailedError`), or a 409 attempt-supersession (`HostedAttemptSupersededError`) + was observed on any outbound call -- the run stops launching further scenarios the moment it is + set. The caller maps this to exit code 3 and must not call `emit_skipped_receipts` (no further + outbound emission once fenced).""" + + receipts: tuple[ResultReceipt, ...] + aborted: ReceiptFailure | None + fenced: BaseException | None = None + + +def _skipped_receipt(scenario: Scenario) -> ResultReceipt: + # Exact body per outbound-channels.md Channel 2, "skipped receipt body (exact)". + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=1, + world_index=None, + status="skipped", + sub_goals=(), + evaluations=(), + call=None, + failure=None, + ) + + +def _unjudged(sub_goals: Sequence[SubGoal]) -> tuple[SubGoalResult, ...]: + return tuple( + # R11: outbound-channels.md pins `judged` as `SubGoal.judged != ""`, not `bool(...)` — + # they agree for every `str` but `bool` is not what the contract names. + SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "") + for goal in sub_goals + ) + + +_LEAK_HEADROOM = 10 # R1: spine §1's hosted `scenario_count` admission range is 1..10 -- the most +# phase threads that can ever be simultaneously abandoned (leaked) in one job. + + +def _abort_from_no_worlds(exc: NoWorldsAvailable) -> ReceiptFailure: + # A uniform §2f never-retried code across every unhealthy world surfaces AS that code+domain; + # otherwise this is the generic exhaustion abort. + if exc.code is not None and exc.domain is not None: + return ReceiptFailure( + domain=exc.domain.value, stage=HarnessStage.RUNNING.value, code=exc.code, message=_truncate(str(exc)) + ) + return _failure("world_pool_exhausted", str(exc)) + + +@dataclass +class _ScenarioContext: + """R7: `_run_scenario` records the world/attempt it is currently working on here as it goes, + so a crash that escapes every handled path still lets `worker()` report the REAL + world_index/scenario_attempt on the `driver_crashed` receipt instead of always None/1. + `call` is set the moment the call step returns, so a LATER crash (e.g. `read_only()` + building the check-phase handle) still reports the call that genuinely ran, not `null`.""" + + world_index: int | None = None + attempt: int = 1 + call: CallSummary | None = None + + +@dataclass(frozen=True) +class _PendingRetryReceipt: + """R3: carries attempt-1's already-built `_Retry` outcome across the retry-lease boundary, so + a cancel/abort landing anywhere between "attempt 1 finished" and "attempt 2 actually starts" + still reports what attempt 1 produced instead of losing it to skipped-synthesis (the same + defect M8 fixed on the `NoWorldsAvailable` branch, on the other post-attempt-1 exit).""" + + world_index: int + attempt: int + outcome: "_Retry" + + +class HostedScheduler: + """Drains a job's scenario list across a `WorldPool`, one asyncio task per scenario — lease() + blocking when the pool is saturated is what caps concurrency at W, so nothing here re-derives + a worker count. Retry is fixed at one extra attempt on a fresh world (spine §5 step 4), gated + on `FailureDomain` per the P9 brief: retryable domains retry once, deterministic ones do not. + """ + + def __init__( + self, + *, + pool: WorldPool, + world_factory: WorldFactory, + call_runner: CallRunner, + outbound: OutboundPort, + job_seed: int, + cancel_requested: Callable[[], bool] | None = None, + ) -> None: + self._pool = pool + self._world_factory = world_factory + self._call_runner = call_runner + self._outbound = outbound + self._job_seed = job_seed + self._cancel_requested = cancel_requested or (lambda: False) + self._executor: ThreadPoolExecutor | None = None + + async def run(self, scenarios: Sequence[Scenario]) -> RunResult: + results: list[ResultReceipt | None] = [None] * len(scenarios) + abort_holder: list[ReceiptFailure | None] = [None] + + # R1: a dedicated executor for scenario phase threads — never the loop's default + # executor, which the provider's own `to_thread` calls (process_runtime.py) also use, and + # whose capacity a leaked phase thread would starve globally. One worker per live world + # plus headroom for the worst case of every admitted scenario leaking its own abandoned + # thread at once (world-handle-interface.md: "its thread leaks, bounded by scenario + # count"). + self._executor = ThreadPoolExecutor( + # `_LEAK_HEADROOM` alone assumes spine §1's `scenario_count` admission cap (<=10) — + # widen for whatever `scenarios` actually holds, or an over-cap job's overflow + # scenarios find the executor saturated and report `driver_crashed` for a phase that + # was queued, not run. + max_workers=max(self._pool.effective_size + _LEAK_HEADROOM, len(scenarios) + 1), + thread_name_prefix="hosted-scenario", + ) + try: + + async def worker(index: int, scenario: Scenario) -> None: + # `self._pool.fenced` is the same stop-path as `abort_holder`/`cancel_requested` + # -- once any outbound call has hit a 401/403 or an exhausted channel, no further + # scenario may even start. + if abort_holder[0] is not None or self._pool.fenced is not None or self._cancel_requested(): + return + context = _ScenarioContext() + try: + results[index] = await self._run_scenario( + scenario, index, abort_holder=abort_holder, context=context + ) + except NoWorldsAvailable as exc: + abort_holder[0] = _abort_from_no_worlds(exc) + except _FATAL_OUTBOUND: + # Already latched onto `self._pool.fenced` by whichever `_emit`/`_log` call + # raised it -- no receipt for a scenario the platform already superseded. + pass + except asyncio.CancelledError: + raise + except BaseException as exc: # noqa: BLE001 + # B3: the scheduler's own machinery crashing must not suppress every other + # scenario's receipt — `gather(return_exceptions=True)` below is the second + # half of that guarantee. + results[index] = await self._driver_crashed_receipt( + scenario, exc, world_index=context.world_index, scenario_attempt=context.attempt, + call=context.call, + ) + + tasks = [asyncio.create_task(worker(i, s)) for i, s in enumerate(scenarios)] + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + receipts: list[ResultReceipt] = [] + for index, scenario in enumerate(scenarios): + receipt = results[index] + if receipt is None: + # (outbound-channels.md v1.3 Sequencing: "terminal event -> skipped + # receipts -> manifest"): only SYNTHESIZE here. `run()` returns before its + # caller has emitted a terminal event, so pushing this over `outbound` now + # would put it on the wire ahead of the terminal -- `emit_skipped_receipts()` + # is the caller's job, done AFTER its own terminal event. + receipt = _skipped_receipt(scenario) + receipts.append(receipt) + return RunResult(receipts=tuple(receipts), aborted=abort_holder[0], fenced=self._pool.fenced) + finally: + # R1: never block `run()` on abandoned threads — `shutdown(wait=True)` would hang + # this coroutine exactly like the bug this fixes. Queued-but-unstarted work is + # cancelled; already-running (leaked) threads are the contract's own accepted, + # bounded tradeoff (world-handle-interface.md's "the job TTL is the backstop"). + self._executor.shutdown(wait=False, cancel_futures=True) + + async def emit_skipped_receipts(self, result: RunResult) -> None: + """R6: outbound-channels.md v1.3 Sequencing — "terminal event -> skipped receipts -> + manifest". `run()` only synthesizes `skipped` receipts into `RunResult.receipts`; call + this AFTER the caller's own terminal event has been emitted, never before, and exactly + once — each call re-emits every `skipped` receipt in `result.receipts` with no dedup of + its own. + + A no-op once `self._pool.fenced` is set (checked live, so it also covers a fence that + landed after `run()` returned but before this call) — the run stopped emitting the moment + the fence was observed and must not resume for these. If a fence instead lands DURING this + method's own loop, the same `_FATAL_OUTBOUND` that stops `run()` escapes out of this method + too; the caller must be ready for that.""" + if self._pool.fenced is not None: + return + for receipt in result.receipts: + if receipt.status == "skipped": + await self._emit(self._outbound.receipt(receipt), what="receipt") + + async def _emit(self, awaitable: Awaitable[None], *, what: str) -> None: + # B3: `OutboundPort` exceptions are best-effort telemetry — never receipt-affecting and + # never fatal to the run. Logged through the same port when logging itself doesn't also + # fail; swallowed otherwise rather than let a transport hiccup kill the scenario loop. + # The one exception besides `CancelledError` this deliberately does NOT swallow — a + # fence (401/403) or an exhausted channel (404x3) is never best-effort telemetry. + try: + await awaitable + except _FATAL_OUTBOUND as exc: + self._pool.mark_fenced(exc) + raise + except Exception as exc: # noqa: BLE001 + try: + await self._outbound.log(level="error", message=f"outbound.{what} failed: {exc}") + except _FATAL_OUTBOUND as log_exc: + self._pool.mark_fenced(log_exc) + raise + except Exception: # noqa: BLE001 + pass + + async def _driver_crashed_receipt( + self, + scenario: Scenario, + exc: BaseException, + *, + world_index: int | None, + scenario_attempt: int, + call: CallSummary | None = None, + ) -> ResultReceipt: + failure = _failure("driver_crashed", f"{type(exc).__name__}: {exc}") + try: + # R7: best-effort — every declared goal, `held: null`, matching the errored-receipt + # body's rule. Falls back to `()` only when reading `sub_goals` itself is what crashed + # (the one case with no goal list to report at all). + sub_goals = _unjudged(scenario.sub_goals) + except Exception: # noqa: BLE001 + sub_goals = () + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=scenario_attempt, + world_index=world_index, + status="errored", + sub_goals=sub_goals, + evaluations=(), + call=call, # the call step's own summary, if it had already returned when this crashed + failure=failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + async def _emit_pending_retry_receipt( + self, scenario: Scenario, pending: "_PendingRetryReceipt" + ) -> ResultReceipt: + # R3: the single shape both post-attempt-1 "never got to run attempt 2" exits emit. + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=pending.attempt, + world_index=pending.world_index, + status="errored", + sub_goals=pending.outcome.sub_goals, + evaluations=(), + call=pending.outcome.call, + failure=pending.outcome.failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + async def _lease_or_abandon( + self, *, exclude: frozenset[int], abort_holder: list[ReceiptFailure | None] + ) -> tuple[int, EnvironmentRuntime] | None: + def _abandon() -> bool: + # A scenario already queued in `lease()` must also abandon once fenced -- the + # worker-top check alone only stops scenarios that had not started yet. + return abort_holder[0] is not None or self._pool.fenced is not None or self._cancel_requested() + + return await self._pool.lease(exclude=exclude, abandon=_abandon) + + async def _run_scenario( + self, + scenario: Scenario, + scenario_index: int, + *, + attempt: int = 1, + tried: frozenset[int] = frozenset(), + pre_leased: tuple[int, EnvironmentRuntime] | None = None, + pending_retry: "_PendingRetryReceipt | None" = None, + abort_holder: list[ReceiptFailure | None], + context: "_ScenarioContext", + ) -> ResultReceipt | None: + if pre_leased is not None: + world_index, runtime = pre_leased + else: + leased = await self._lease_or_abandon(exclude=tried, abort_holder=abort_holder) + if leased is None: + return None # B5: cancelled/aborted while queued — never got a world + world_index, runtime = leased + + context.world_index = world_index # R7: the real values for a driver_crashed receipt + context.attempt = attempt + context.call = None # this attempt has not made its own call yet -- must not still + # carry a previous attempt's summary on the shared context object into this one's receipt. + + # B5: re-check immediately after `lease()` returns — a cancel/abort landing while this + # worker was queued must not let a freshly granted world start work it can never finish + # inside the flush window. + if abort_holder[0] is not None or self._cancel_requested(): + await self._pool.release(world_index) + if pending_retry is not None: + # R3: attempt 1 already ran on `pending_retry.world_index` and produced a real + # outcome — this is the retry continuation (this world was never used for it). + return await self._emit_pending_retry_receipt(scenario, pending_retry) + return None + + world_resolved = False # B3: the leased world must be released/discarded exactly once + try: + if pending_retry is not None: + # Emitted here, immediately before attempt 2's own `scenario_started`, so this + # event and the pending-retry receipt (both exits above) are mutually exclusive + # by construction — outbound-channels.md Channel 2: "the failed first try is + # recorded by scenario_retried/world_unhealthy events, never by a receipt." + await self._emit( + self._outbound.scenario_retried( + scenario_key=scenario.scenario_key, + from_world=pending_retry.world_index, + to_world=world_index, + ), + what="scenario_retried", + ) + await self._emit( + self._outbound.scenario_started( + scenario_key=scenario.scenario_key, + world_index=world_index, + scenario_attempt=attempt, + ), + what="scenario_started", + ) + + rng = random.Random(self._job_seed + scenario_index) + outcome: ResultReceipt | _Retry + try: + world = await self._world_factory.create(runtime, rng=rng) + except Exception as exc: # noqa: BLE001 + # B3: `world_factory.create()` failing (e.g. a PostgresStore connect failure) is + # the same shape as a mid-scenario `WorldUnavailable` — the world is unusable, not + # the scenario code. Deliberately narrow to just this call: `_execute()` has its + # own exhaustive internal exception handling (`_run_phase`/`_invoke`), so anything + # that still escapes it is a genuine scheduler bug and belongs in `driver_crashed` + # (via `worker()`'s `BaseException` catch), not swallowed into `world_unavailable`. + outcome = _Retry( + _failure("world_unavailable", f"{type(exc).__name__}: {exc}"), + sub_goals=_unjudged(scenario.sub_goals), + call=None, + mark_unhealthy=True, + ) + else: + outcome = await self._execute( + scenario, world, runtime, world_index, attempt=attempt, context=context + ) + + if isinstance(outcome, _Retry): + # R6: `mark_unhealthy()` itself emits `world_unhealthy` now (every demotion path + # goes through it) — no separate emit needed here. + if outcome.mark_unhealthy: + await self._pool.mark_unhealthy(world_index, cause=outcome.failure.message) + else: + await self._pool.release(world_index) + world_resolved = True + + if attempt >= 2: + receipt = ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status="errored", + sub_goals=outcome.sub_goals, + evaluations=(), + call=outcome.call, + failure=outcome.failure, + ) + await self._emit(self._outbound.receipt(receipt), what="receipt") + return receipt + + # R3: attempt 1's outcome, carried forward so either exit below that never gets to + # start attempt 2 can still report it instead of losing it to skipped-synthesis. + pending = _PendingRetryReceipt(world_index=world_index, attempt=attempt, outcome=outcome) + try: + next_leased = await self._lease_or_abandon( + exclude=tried | {world_index}, abort_holder=abort_holder + ) + except NoWorldsAvailable as exc: + # M8: this scenario already ran and produced a real attempt-1 failure — losing + # it to skipped-synthesis just because the retry lease found nothing would + # report "never ran" for a scenario that manifestly did. + abort_holder[0] = _abort_from_no_worlds(exc) # v1.13 §5.4 + return await self._emit_pending_retry_receipt(scenario, pending) + + if next_leased is None: + # R3: same defect as the branch above, reached via cancel/abort instead of + # pool exhaustion. + return await self._emit_pending_retry_receipt(scenario, pending) + next_index, next_runtime = next_leased + return await self._run_scenario( + scenario, + scenario_index, + attempt=2, + tried=tried | {world_index}, + pre_leased=(next_index, next_runtime), + pending_retry=pending, + abort_holder=abort_holder, + context=context, + ) + + # M13: a plain terminal receipt is either a real passed/failed verdict (release — the + # world is fine) or a non-retryable fault from `_fault()`. For the latter, an + # exception/overrun code means the world is half-applied and must be discarded rather + # than handed to the next scenario; `ready_not_ready` is a clean verdict and keeps + # `release()`. + if outcome.failure is not None and outcome.failure.code in _DISCARD_ON_ERROR_CODES: + await self._pool.mark_unhealthy(world_index, cause=outcome.failure.message) + else: + await self._pool.release(world_index) + world_resolved = True + await self._emit(self._outbound.receipt(outcome), what="receipt") + return outcome + finally: + if not world_resolved: + if self._pool.fenced is not None: + # The exception that skipped every path above was a fence (or the pool was + # already fenced by something else) -- the world itself never did anything + # wrong. `mark_unhealthy()` here would emit a false `world_unhealthy` after the + # run already stopped emitting, and schedule a `provision()` reconcile for a + # job that is not coming back for it. + await self._pool.release(world_index) + else: + # Something blew past every handled path above (a bug in this module + # itself) — the world must not be silently stranded outside the pool's + # bookkeeping. Discarded rather than released: an exception here leaves its + # state unknown, and world-handle-interface.md's own exception rule is + # "discarded and re-provisioned, never reused." + await self._pool.mark_unhealthy( + world_index, cause="scenario driver crashed while holding this world" + ) + + async def _execute( + self, + scenario: Scenario, + world: World, + runtime: EnvironmentRuntime, + world_index: int, + *, + attempt: int, + context: "_ScenarioContext", + ) -> "ResultReceipt | _Retry": + setup = await _run_phase( + scenario.setup, world, timeout=SETUP_TIMEOUT_SECONDS, phase="setup", executor=self._executor + ) + if setup.failure is not None: + return self._fault(scenario, world_index, attempt, setup.failure, sub_goals=_unjudged(scenario.sub_goals)) + + read_only = world.read_only() + ready = await _run_phase( + scenario.ready, read_only, timeout=READY_TIMEOUT_SECONDS, phase="ready", executor=self._executor + ) + if ready.failure is not None: + return self._fault(scenario, world_index, attempt, ready.failure, sub_goals=_unjudged(scenario.sub_goals)) + verdict = _classify_ready(ready.value) + if verdict.broken: + return self._fault( + scenario, world_index, attempt, _failure("ready_broken", f"ready() returned {ready.value!r}"), + sub_goals=_unjudged(scenario.sub_goals), + ) + if not verdict.held: + return self._fault( + scenario, world_index, attempt, _failure("ready_not_ready", verdict.reason or ""), + sub_goals=_unjudged(scenario.sub_goals), + ) + + try: + call_outcome = await self._call_runner.run(scenario, runtime) + except WorldUnavailable as exc: + return _Retry(_failure("world_unavailable", str(exc)), sub_goals=_unjudged(scenario.sub_goals), call=None, mark_unhealthy=True) + except CallAborted as exc: + call = self._call_summary(exc.partial) + return self._fault( + scenario, world_index, attempt, _failure("call_failed", str(exc)), + sub_goals=_unjudged(scenario.sub_goals), call=call, + ) + except Exception as exc: # noqa: BLE001 + # B3: the call runner crashing outright (not a `CallAborted` it chose to raise) is the + # same world-handle-interface.md v3.3 row — "the simulated-call machinery crashed" — + # just with no partial evidence to report. + return self._fault( + scenario, world_index, attempt, _failure("call_failed", f"{type(exc).__name__}: {exc}"), + sub_goals=_unjudged(scenario.sub_goals), call=None, + ) + + # Set the moment the call step returns, so a crash later in this method (e.g. + # `world.read_only()` below) still reports the call that genuinely ran, not `null`. + context.call = self._call_summary(call_outcome) + calls = list(call_outcome.calls) # m12: `folder.py::_RUNNABLE` expects a list, not a tuple. + if not calls: + # M10: unconditioned on `turns` — an empty list must never reach checks regardless of + # whether the simulator observed a turn (world-handle-interface.md "Coverage + # guarantee": "An empty list is never handed to checks"). + failure = _failure("evidence_missing", "no tool calls were captured for this scenario's call") + return _Retry(failure, sub_goals=_unjudged(scenario.sub_goals), call=self._call_summary(call_outcome), mark_unhealthy=False) + + if not scenario.sub_goals: + # m7: `all(())` is vacuously True — a scenario declaring zero sub-goals must not read + # as a silent pass. + return self._fault( + scenario, world_index, attempt, + _failure("check_broken", "scenario declared zero sub_goals — a vacuous pass is forbidden"), + sub_goals=(), call=self._call_summary(call_outcome), + ) + + sub_goal_results: list[SubGoalResult] = [] + check_handle = world.read_only() + broken_failure: ReceiptFailure | None = None + for goal in scenario.sub_goals: + if broken_failure is not None: + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + outcome = await _run_phase( + goal.check, check_handle, calls, timeout=CHECK_TIMEOUT_SECONDS, phase="check", executor=self._executor + ) + if outcome.failure is not None: + if outcome.failure.code == "world_unavailable": + return _Retry( + outcome.failure, sub_goals=tuple(sub_goal_results) + _unjudged([goal]) + _unjudged(scenario.sub_goals[len(sub_goal_results) + 1 :]), + call=self._call_summary(call_outcome), mark_unhealthy=True, + ) + broken_failure = outcome.failure + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + verdict = _classify_check(outcome.value) + if verdict.broken: + broken_failure = _failure("check_broken", f"{goal.name}: check() returned {outcome.value!r}") + sub_goal_results.append(SubGoalResult(name=goal.name, held=None, reason=None, judged=goal.judged != "")) + continue + sub_goal_results.append(SubGoalResult(name=goal.name, held=verdict.held, reason=verdict.reason, judged=goal.judged != "")) + + if broken_failure is not None: + return self._fault( + scenario, world_index, attempt, broken_failure, sub_goals=tuple(sub_goal_results), call=self._call_summary(call_outcome), + ) + + status = "passed" if all(result.held for result in sub_goal_results) else "failed" + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status=status, + sub_goals=tuple(sub_goal_results), + evaluations=(), + call=self._call_summary(call_outcome), + failure=None, + ) + + @staticmethod + def _call_summary(outcome: CallOutcome | None) -> CallSummary | None: + if outcome is None: + return None + return CallSummary( + started_at=outcome.started_at, + ended_at=outcome.ended_at, + duration_ms=outcome.duration_ms, + turns=outcome.turns, + transcript_artifact=outcome.transcript_artifact, + recording_artifacts=outcome.recording_artifacts, + ) + + def _fault( + self, + scenario: Scenario, + world_index: int, + attempt: int, + failure: ReceiptFailure, + *, + sub_goals: tuple[SubGoalResult, ...], + call: CallSummary | None = None, + retry: bool | None = None, + ) -> "ResultReceipt | _Retry": + should_retry = _is_retryable(failure.code) if retry is None else retry + if should_retry: + return _Retry(failure, sub_goals=sub_goals, call=call, mark_unhealthy=failure.code == "world_unavailable") + return ResultReceipt( + scenario_key=scenario.scenario_key, + scenario_id=scenario.scenario_id, + scenario_attempt=attempt, + world_index=world_index, + status="errored", + sub_goals=sub_goals, + evaluations=(), + call=call, + failure=failure, + ) + + +@dataclass(frozen=True) +class _Retry: + failure: ReceiptFailure + sub_goals: tuple[SubGoalResult, ...] + call: CallSummary | None + mark_unhealthy: bool + + +__all__ = [ + "CHECK_TIMEOUT_SECONDS", + "READY_TIMEOUT_SECONDS", + "SETUP_TIMEOUT_SECONDS", + "Call", + "CallAborted", + "CallOutcome", + "CallRunner", + "CallSummary", + "Evaluation", + "HostedScheduler", + "NoWorldsAvailable", + "OutboundPort", + "ReadOnlyWorld", + "ReceiptFailure", + "ResultReceipt", + "RunResult", + "Scenario", + "SubGoal", + "SubGoalResult", + "World", + "WorldFactory", + "WorldPool", + "WorldProvisioner", +] diff --git a/src/fi/alk/harness/outbound.py b/src/fi/alk/harness/outbound.py new file mode 100644 index 00000000..b0fe7d3b --- /dev/null +++ b/src/fi/alk/harness/outbound.py @@ -0,0 +1,2960 @@ +"""Outbound reporting — `outbound-channels.md` v1.3, paired with `hosted-execution-seams.md` v1.11 +(the spine). All guest -> platform traffic (events, result receipts, artifacts) is outbound HTTPS; +the platform never calls in. Two halves: + +Foundations (part 1): the platform capability declaration the gateway uploads to +`/run/futureagi/capabilities.json`, the byte-exact canonical serialization every digest in the +contract is built from, and the durable local spool (with its monotonic sequence allocator) that +emission sits behind so a killed process within a live sandbox never loses or duplicates a +record. (A killed sandbox is deleted by the gateway, spool and all -- there is no in-sandbox +restart producer in the spine today, so the cross-restart recovery machinery guards a scenario +that isn't triggerable yet, but the in-process failed-write and watermark-durability guarantees +are load-bearing from the first event.) + +Transport (part 2): a channel-neutral `Transport` protocol plus a `requests`-backed production +implementation; the closed status-code error map (`classify_response`) shared by all three channel +clients; and the clients themselves — `EventsClient` (batches spooled events, advances the spool +watermark only on confirmed delivery), `ResultsClient` (typed `ResultReceiptDraft` + delivery), and +`ArtifactsClient` (content-addressed upload + `ArtifactManifestDraft` + delivery), each sharing one +retry/backoff engine (`_perform_with_retry`) that raises on the two channel-ending outcomes +(`HostedFencedError` for 401/403, `HostedChannelFailedError` for 404 exhausted) and returns a typed +`ChannelError` for everything else a caller must log-and-continue on. + +`HostedEvent`/`HostedEventDraft` model Channel 1's wire shape (the "hosted event model" the spine's +implementation-delta list calls for — `sequence`/`attempt_id`/`attempt_number`/`stage`/`digest` — +distinct from `fi.simulate.runtime.events.CanonicalEvent`, which remains the local-SDK wire and is +untouched by this module). + +Redaction (v1.3 Channel 1; seams v1.11 §3): `redact_outbound_text` scrubs URL userinfo +(`scheme://user:pw@` -> `scheme://user:***@`) plus an adapter-supplied secret-value list, applied +inside `build_event_record`/`build_result_receipt` to every free-text field the contract names +(`log.message`, `world_unhealthy.cause`, `terminal`/receipt `failure.message`, sub_goal/evaluation +`reason`). It is NOT the full "same secret-content scan as the artifact sealer" the contract also +requires — that broader scan is a separate, sealer-side obligation this module does not implement. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import os +import random +import re +import time +import uuid +from collections.abc import Callable, Collection, Iterator, Mapping +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from threading import RLock +from typing import Annotated, Any, ClassVar, Literal, Protocol +from urllib.parse import urlparse + +try: # N30: fcntl is POSIX-only; the guest is Linux-only, but the module must still IMPORT + import fcntl +except ImportError: # pragma: no cover - non-POSIX + fcntl = None # type: ignore[assignment] + +import requests +from pydantic import ( + AfterValidator, + BaseModel, + ConfigDict, + Field, + JsonValue, + ValidationError, + field_serializer, + model_validator, +) + +from .job import FailureDomain, HarnessStage + +logger = logging.getLogger(__name__) + +CAPABILITIES_SCHEMA_VERSION = "futureagi.harness-capabilities.v1" +CAPABILITIES_PATH = "/run/futureagi/capabilities.json" +EVENT_SCHEMA_VERSION = "futureagi.harness-event.v1" +RESULT_SCHEMA_VERSION = "futureagi.harness-result.v1" +MANIFEST_SCHEMA_VERSION = "futureagi.harness-manifest.v1" + +# "Channel 1" limits (outbound-channels.md v1.3) that every producer of an event record needs +# to honor before it ever reaches a transport client. MIN-12: consumed by EventsClient.flush() +# (stamps `schema_version`, clamps the batch to EVENTS_MAX_BATCH) and by HostedEventDraft's own +# size check -- no longer just declared and unused. +EVENTS_MAX_BATCH = 100 +EVENT_PAYLOAD_MAX_BYTES = 32 * 1024 +# N7: a cumulative-bytes cap on top of EVENTS_MAX_BATCH's event-count cap. 100 events * 32KB could +# reach ~3.2MB; this keeps a proactively-built batch comfortably under a common ~1MB ingress cap +# (nginx's default) so 413 is the exception, not the steady state -- EventsClient.flush() still +# halves and retries reactively on an observed 413 regardless of this cap. +EVENTS_MAX_BATCH_BYTES = 900_000 +# §3a: uploads over this size use chunked transfer; consumed by ArtifactsClient's default +# chunk_threshold_bytes. +ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES = 64 * 1024 * 1024 +# "Sequencing"/"Flush window": the drain deadline from the cancel signal, TTL, or terminal event. +# N5/N24: this module does not compute a deadline from it -- every public client method +# (EventsClient.flush / ResultsClient.push / ArtifactsClient.upload / .push_manifest) instead +# accepts an explicit `deadline: float | None` (a `time.monotonic()` value). The adapter (P10) is +# the one process-wide owner of "when did the window start," so it is the one that turns this +# constant into the deadline value it passes in -- not this module. +FLUSH_WINDOW_SECONDS = 120 + + +# ================================================================================================= +# Canonicalization -- the byte-exact serialization every digest in the contract is built from. +# ================================================================================================= + + +class OutboundError(RuntimeError): + """Generic typed failure for this module's canonicalization/digest layer -- same `code`/ + `message` shape as `CapabilitiesError`/`OutboundSpoolError`, used where neither of those is the + right domain (canonicalization itself, and shape checks that run before any spool or + capabilities object exists).""" + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +def canonical_bytes(value: Any) -> bytes: + """The contract's canonical form ("Canonicalization (every digest in this file)"): + ``json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False)``, + encoded UTF-8 (v1.3 pins `allow_nan=False` explicitly). This is the ONLY place that call is + made -- every digest function in this module goes through it, so a change to the algorithm + cannot happen in only one of them. + + `allow_nan=False` changes nothing about the bytes for any value that was already valid JSON -- + NaN/Infinity are not RFC 8259, so a float that would have silently produced unparseable bytes + now fails loudly here instead of downstream at the platform's parser. + + Never re-derive an already-spooled record's bytes by calling this again on retry: a dict's key + order is stable within one process but nothing guarantees float formatting or dict construction + order is bit-identical across a restart. `OutboundSpool` hands back the literal bytes it wrote; + those are what a retry re-sends, per the contract's "serialize once, spool the bytes, re-send + verbatim; never re-serialize on retry." + """ + try: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False + ).encode("utf-8") + except ValueError as exc: + raise OutboundError("canonical_value_not_finite", str(exc)) from exc + + +def sha256_digest(data: bytes) -> str: + return "sha256:" + hashlib.sha256(data).hexdigest() + + +def event_payload_digest(payload: dict[str, Any]) -> str: + """Event digest scope: the `payload` object alone (not the envelope around it).""" + return sha256_digest(canonical_bytes(payload)) + + +def _json_native_offense(value: Any, path: str) -> str | None: + """Walks `value` looking for the first thing `canonical_bytes` cannot represent for a reason + OTHER than NaN/Infinity (which `canonical_bytes` itself catches): a non-JSON-native Python + value (`datetime`, `Decimal`, `UUID`, ...) or a non-string dict key. Returns the offending key + path (e.g. `"call.started_at"`), or `None` if the tree is clean.""" + if value is None or isinstance(value, (bool, int, float, str)): + return None + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + return f"{path}[:{key!r}]" if path else f":{key!r}" + offense = _json_native_offense(item, f"{path}.{key}" if path else key) + if offense is not None: + return offense + return None + if isinstance(value, list): + for index, item in enumerate(value): + offense = _json_native_offense(item, f"{path}[{index}]") + if offense is not None: + return offense + return None + return path or "" + + +def whole_object_digest(obj: dict[str, Any]) -> str: + """Receipt/manifest digest scope: the whole object with the `digest` key ABSENT. + + The key is popped, never set to `None` -- the contract is explicit that "absent and null are + different bytes," so silently keeping `digest: null` in the canonicalized form would compute a + different (wrong) hash than what the platform verifies against. + + Unlike an event payload (already pydantic-validated as `dict[str, JsonValue]` before it ever + reaches `event_payload_digest`), receipts and manifests reach this function as hand-built + dicts from whatever calls it -- a bare `TypeError` from `json.dumps` on a non-JSON-native value + or a non-string key is a debugging dead end with no indication of WHERE in the object the bad + value lives. This walks the tree first and raises a typed `OutboundError` naming the offending + key path instead. + """ + core = {key: value for key, value in obj.items() if key != "digest"} + offense = _json_native_offense(core, "") + if offense is not None: + raise OutboundError("digest_value_not_json_native", f"non-JSON-native value at: {offense}") + return sha256_digest(canonical_bytes(core)) + + +_DIGEST_PATTERN = re.compile(r"sha256:[0-9a-f]{64}") + + +def is_valid_digest(value: str) -> bool: + return bool(_DIGEST_PATTERN.fullmatch(value)) + + +_RFC3339_MILLIS_PATTERN = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z") + + +def format_rfc3339_millis(value: datetime) -> str: + """The contract's exact timestamp wire form ("Timestamps: RFC 3339, UTC, `Z`, millisecond + precision"). Unlike `HostedEvent.emitted_at` (an envelope field the event digest scope never + covers, so its exact string form doesn't matter), receipt/manifest timestamps such as + `call.started_at` sit INSIDE the `whole_object_digest` scope -- what we hash must be byte- + identical to what we send, so those fields are plain `str` on the wire models, produced only + through this function, never through a datetime's default serialization (which pydantic would + render as e.g. `+00:00` offset and six-digit microseconds, not `Z` and milliseconds). + """ + if value.tzinfo is None: + raise ValueError("naive_datetime_not_allowed") + utc = value.astimezone(timezone.utc) + return utc.strftime("%Y-%m-%dT%H:%M:%S.") + f"{utc.microsecond // 1000:03d}Z" + + +def is_valid_rfc3339_millis(value: str) -> bool: + return bool(_RFC3339_MILLIS_PATTERN.fullmatch(value)) + + +def _require_utc_millis(value: datetime) -> datetime: + """Shared `AfterValidator` for every `datetime` field this module serializes through + `format_rfc3339_millis` (`HostedEventDraft.emitted_at`, `HostedCapabilities.expires_at`): + rejects a naive datetime outright (the contract's wire form has no naive representation), + converts any other offset to UTC, and truncates to millisecond precision so the VALUE itself -- + not just its string rendering -- matches what gets sent on the wire.""" + if value.tzinfo is None: + raise ValueError("naive_datetime_not_allowed") + utc = value.astimezone(timezone.utc) + return utc.replace(microsecond=(utc.microsecond // 1000) * 1000) + + +UtcMillisDatetime = Annotated[datetime, AfterValidator(_require_utc_millis)] + + +# N9/P3: URL userinfo -- `scheme://user:pw@host` -> `scheme://user:***@host`, matching the seams +# contract's own example (`postgresql://harness:***@...`). The username group is now OPTIONAL +# (`redis://:pw@host`, the canonical empty-username shape for Redis/RabbitMQ/Mongo, is a real +# managed-store DSN form) and so is the whole password group (`https://@host`, the standard +# way a bearer token appears in git/registry output) -- a bare userinfo token is masked outright +# rather than left verbatim on the theory that "a username alone is not a secret," which is false +# for a token. +_USERINFO_PATTERN = re.compile(r"([a-zA-Z][a-zA-Z0-9+.\-]*://)([^\s:/?#@]*)(:[^\s/?#]*)?@") + + +def _mask_userinfo(match: re.Match[str]) -> str: + scheme, user, password = match.group(1), match.group(2), match.group(3) + return f"{scheme}{user}:***@" if password is not None else f"{scheme}***@" + + +def redact_outbound_text(value: str, extra_secret_values: tuple[str, ...] = ()) -> str: + """Scrubs a single free-text field before it can leave the sandbox on any of the three + channels (outbound-channels.md v1.3 "Redaction (enforced before emit)"; hosted-execution- + seams.md v1.11 §3 "any outbound projection ... redacts userinfo"). Two things, applied in + order: + + 1. URL userinfo (`_USERINFO_PATTERN`/`_mask_userinfo`, above) -- a password is masked and the + username kept (matching the contract's own `postgresql://harness:***@...` example); a bare + token/username-only userinfo (no `:`) is masked outright, since that shape is how a bearer + token appears, not a username. + 2. `extra_secret_values` -- exact-substring replacement for a caller-supplied list of secret + values. Always `()` today: the adapter (P10) is what will know the job's declared secrets + and pass them in -- this parameter exists now so `build_event_record`/`build_result_receipt` + never need to change shape when that wiring lands. + + NOT a general secret-content scanner -- the contract's "same secret-content scan as the + artifact sealer" is a separate, sealer-side obligation. This is the narrow subset this module + can enforce on every string field it controls without false-positiving on ordinary diagnostic + text. + """ + redacted = _USERINFO_PATTERN.sub(_mask_userinfo, value) + for secret in extra_secret_values: + if secret: + redacted = redacted.replace(secret, "***") + return redacted + + +# ================================================================================================= +# Capabilities -- `/run/futureagi/capabilities.json`, "Authentication" section. +# ================================================================================================= + + +class CapabilitiesError(RuntimeError): + """A capability declaration is missing, malformed, or fails a shape rule. + + Mirrors the `code`/`message` shape `BundleV2Error`/`PreflightError` use elsewhere in this + package. `code` is one of the nine closed values in outbound-channels.md v1.3's "Capabilities- + file rejection table" -- see `load_capabilities`, which is the sole place that maps a raw + failure onto one of them. + """ + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +class HostedEndpoints(BaseModel): + """The four outbound routes, per attempt. Shape-only checks (trailing slash, https) live here + as defense-in-depth for direct construction; `load_capabilities` runs the SAME checks earlier, + outside pydantic, so each has its own `CapabilitiesError.code` instead of collapsing into the + generic `capabilities_field_invalid` (see v1.3's rejection table).""" + + model_config = ConfigDict(extra="forbid") + + events: str + results: str + artifacts: str + scenarios: str + + @model_validator(mode="after") + def _shape(self) -> "HostedEndpoints": + for name in ("events", "results", "artifacts", "scenarios"): + value = getattr(self, name) + if not value or not value.endswith("/"): + raise ValueError(f"{name} endpoint must end with '/'") + if not value.startswith("https://"): + raise ValueError(f"{name} endpoint must use https") + return self + + +class HostedCapabilities(BaseModel): + """The per-attempt bearer plus the four endpoint URLs. Loaded once at emitter startup from the + file the gateway uploads (§0 step 4) -- see `load_capabilities`, which also implements the + contract's "loaded into memory ... and unlinked" lifetime rule.""" + + model_config = ConfigDict(extra="forbid") + + schema_version: str + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + fence: str = Field(min_length=1) + expires_at: UtcMillisDatetime + token: str = Field(min_length=1) + endpoints: HostedEndpoints + + @model_validator(mode="after") + def _schema_shape(self) -> "HostedCapabilities": + # Defense-in-depth only -- `load_capabilities` checks this first, outside pydantic, so it + # can raise `capabilities_schema_unsupported` specifically rather than the generic + # `capabilities_field_invalid` this validator's `ValueError` would collapse into. + if self.schema_version != CAPABILITIES_SCHEMA_VERSION: + raise ValueError(f"unsupported schema_version: {self.schema_version}") + return self + + @field_serializer("expires_at") + def _serialize_expires_at(self, value: datetime) -> str: + return format_rfc3339_millis(value) + + def auth_headers(self) -> dict[str, str]: + """"Every request: `Authorization: Bearer ` + `X-Harness-Fence: `." Pure + formatting -- issuing the request itself is a P8 transport-client concern.""" + return {"Authorization": f"Bearer {self.token}", "X-Harness-Fence": self.fence} + + def event_builder(self, *, extra_secret_values: tuple[str, ...] = ()) -> Callable[..., dict[str, Any]]: + """A `build_event_record`-shaped callable with `job_id`/`attempt_id`/`attempt_number` + closed over from THIS capabilities object. The contract's `403 attempt_mismatch` fires when + an event's identity disagrees with the token authenticating it -- binding these three + fields here makes that class of caller bug unrepresentable at the call site instead of a + runtime 403 discovered mid-attempt. + + P4: `extra_secret_values` is bound here too, alongside identity, rather than left as a + per-call parameter -- `build_event_record` could not previously receive the job's declared + secret list at all through this binder, so a caller wired for `event_builder()` had no way + to satisfy N9's redaction requirement on `log.message`/`terminal.failure.message` without + routing around this method entirely. Binding once here matches how identity is already + bound and gives the adapter one place to get it wrong instead of two. + """ + + def build( + *, + event_id: str, + emitted_at: datetime, + stage: HarnessStage, + type: OutboundEventType, + payload: dict[str, JsonValue], + ) -> dict[str, Any]: + return build_event_record( + event_id=event_id, + job_id=self.job_id, + attempt_id=self.attempt_id, + attempt_number=self.attempt_number, + emitted_at=emitted_at, + stage=stage, + type=type, + payload=payload, + extra_secret_values=extra_secret_values, + ) + + return build + + +def _endpoint_matches_attempt(url: str, attempt_id: str) -> bool: + """"an endpoint's `` path segment disagrees with the declared `attempt_id`" + (`capabilities_attempt_mismatch`, v1.3) -- checked as a whole path segment, not a substring, so + an attempt_id that happens to be a substring of another segment can't produce a false match.""" + segments = [segment for segment in urlparse(url).path.split("/") if segment] + return attempt_id in segments + + +def _redact_validation_error(exc: ValidationError) -> str: + """Builds a `capabilities_field_invalid` message from `loc`/`msg` only -- pydantic's default + `str(exc)` embeds each failing field's `input_value`, and the capabilities file carries the + bearer token; a caller that logs this message must never be able to leak it.""" + parts = [] + for error in exc.errors(): + loc = ".".join(str(part) for part in error.get("loc", ())) + msg = error.get("msg", "") + parts.append(f"{loc}: {msg}" if loc else msg) + return "; ".join(parts) or "capabilities file failed validation" + + +def _warn_if_capabilities_file_insecure(target: Path) -> None: + """"owner svc-control, mode 0600" is the contract's posture for this file, but a wrong mode or + owner is NOT a load-time rejection (MIN-5, fail-safe): the bearer is only a per-attempt token + that expires on its own, and refusing to load it entirely over a permissions mistake would turn + a minor hardening gap into a hard attempt failure. Loud warning only.""" + try: + info = target.stat() + except OSError: + return + mode = info.st_mode & 0o777 + if mode != 0o600: + logger.warning( + "%s: capabilities file mode is %o, expected 0600 -- a world/group-readable bearer in " + "a multi-user sandbox is a leak risk (not blocking the load)", + target, mode, + ) + try: + running_uid = os.geteuid() + except AttributeError: + return # os.geteuid() is POSIX-only + if info.st_uid != running_uid: + logger.warning( + "%s: capabilities file is owned by uid %s, not the running uid %s -- expected owner " + "svc-control per the contract (not blocking the load)", + target, info.st_uid, running_uid, + ) + + +def load_capabilities( + path: str | Path = CAPABILITIES_PATH, + *, + unlink: bool = True, + now: Callable[[], datetime] | None = None, + on_unlink_failure: Callable[[OSError], None] | None = None, +) -> HostedCapabilities: + """Parse and validate one capabilities file against outbound-channels.md v1.3's closed + "Capabilities-file rejection table" (nine codes, all reachable as `CapabilitiesError.code`). + + A guest that cannot load this file has no channel at all -- no token, no endpoints -- so it + cannot report its own failure; every branch below raises before any network-capable object + exists. Most checks run BEFORE `HostedCapabilities.model_validate`, outside any pydantic + validator: wrapping them in a pydantic `ValidationError` (the old shape) meant they only ever + surfaced as the generic `capabilities_field_invalid`, never as their own named code -- checking + here first makes each one an independently raised, independently testable `CapabilitiesError`. + + ``unlink=True`` (the default) implements "loaded into memory at emitter startup and unlinked" -- + the file is only ever removed AFTER a successful parse and validation, never before, so a + crash mid-load leaves the file in place for the next attempt to read rather than destroying the + only copy of a not-yet-consumed bearer. A failure to unlink is never fatal to an otherwise- + successful load (the sandbox is destroyed by the gateway at attempt end regardless) but is no + longer silently swallowed either: it is reported via `on_unlink_failure` if given, else logged + (MIN-7) -- the caller can still tell a 0600 bearer may be lingering on disk. + + ``now`` is injectable (defaults to the real clock) so `capabilities_expired` is testable without + manipulating the wall clock. + """ + target = Path(path).expanduser() + try: + exists = target.is_file() + except OSError as exc: + raise CapabilitiesError("capabilities_file_unreadable", str(exc)) from exc + if not exists: + raise CapabilitiesError("capabilities_file_missing", str(target)) + _warn_if_capabilities_file_insecure(target) + + try: + text = target.read_text(encoding="utf-8") + except OSError as exc: + raise CapabilitiesError("capabilities_file_unreadable", str(exc)) from exc + try: + raw = json.loads(text) + except json.JSONDecodeError as exc: + raise CapabilitiesError("capabilities_file_malformed", str(exc)) from exc + if not isinstance(raw, dict): + raise CapabilitiesError("capabilities_file_malformed", "not a JSON object") + + schema_version = raw.get("schema_version") + if schema_version != CAPABILITIES_SCHEMA_VERSION: + raise CapabilitiesError("capabilities_schema_unsupported", str(schema_version)) + + attempt_id = raw.get("attempt_id") + endpoints_raw = raw.get("endpoints") + if isinstance(endpoints_raw, dict): + for name in ("events", "results", "artifacts", "scenarios"): + value = endpoints_raw.get(name) + if not isinstance(value, str): + continue # missing/wrong-typed -- a shape error pydantic below will catch + if not value.endswith("/"): + raise CapabilitiesError("capabilities_endpoint_invalid", f"endpoints.{name} must end with '/'") + if not value.startswith("https://"): + raise CapabilitiesError("capabilities_endpoint_insecure", f"endpoints.{name} must use https") + if isinstance(attempt_id, str) and attempt_id and not _endpoint_matches_attempt(value, attempt_id): + raise CapabilitiesError( + "capabilities_attempt_mismatch", + f"endpoints.{name} does not carry the declared attempt_id {attempt_id!r}", + ) + + try: + capabilities = HostedCapabilities.model_validate(raw) + except ValidationError as exc: + raise CapabilitiesError("capabilities_field_invalid", _redact_validation_error(exc)) from exc + + current_time = (now or (lambda: datetime.now(timezone.utc)))() + if capabilities.expires_at <= current_time: + raise CapabilitiesError("capabilities_expired", f"expires_at={capabilities.expires_at.isoformat()}") + + if unlink: + try: + target.unlink() + except OSError as exc: + if on_unlink_failure is not None: + on_unlink_failure(exc) + else: + logger.warning( + "%s: failed to unlink the capabilities file after a successful load (%s) -- a " + "0600 bearer may still be on disk; the sandbox is destroyed at attempt end " + "regardless, so this does not fail the load", + target, exc, + ) + return capabilities + + +# ================================================================================================= +# Channel 1 -- Events. The closed `type` vocabulary and each type's payload shape. +# ================================================================================================= + + +class DegradeReason(str, Enum): + CONFORMANCE_GATE_FAILED = "conformance_gate_failed" + FIXED_PORT = "fixed_port" + + +class LogLevel(str, Enum): + DEBUG = "debug" + INFO = "info" + WARNING = "warning" + ERROR = "error" + + +class TerminalReason(str, Enum): + TTL_EXCEEDED = "ttl_exceeded" + USER_CANCELED = "user_canceled" + + +class OutboundEventType(str, Enum): + STAGE_CHANGED = "stage_changed" + PARALLELISM_DEGRADED = "parallelism_degraded" + BASELINE_FROZEN = "baseline_frozen" + BASELINE_INPUTS_CHANGED = "baseline_inputs_changed" + WORLD_UNHEALTHY = "world_unhealthy" + SCENARIO_STARTED = "scenario_started" + SCENARIO_RETRIED = "scenario_retried" + LOG = "log" + TERMINAL = "terminal" + + +class StageChangedPayload(BaseModel): + """No `populate_by_name` -- the wire key is `from` (a Python keyword, hence the `from_stage` + attribute name + alias), and this model is validation-only (`HostedEventDraft` never + normalizes `self.payload`; it emits the caller's dict verbatim). Allowing population by the + attribute name too would let a caller who writes `from_stage` in their payload dict pass + validation while spooling an undefined wire key -- `populate_by_name=True` previously made + exactly that mistake succeed silently.""" + + model_config = ConfigDict(extra="forbid") + + from_stage: HarnessStage | None = Field(alias="from") + to: HarnessStage + + +class ParallelismDegradedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + requested: int = Field(ge=1) + effective: int = Field(ge=1) + reason: DegradeReason + + @model_validator(mode="after") + def _range(self) -> "ParallelismDegradedPayload": + if not (1 <= self.effective < self.requested): + raise ValueError( + f"parallelism_degraded_effective_out_of_range: effective={self.effective} " + f"requested={self.requested}" + ) + return self + + +class BaselineFrozenPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + inputs_digest: str + baseline_ref: str + + +class BaselineInputsChangedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + previous_digest: str | None + current_digest: str + + +class WorldUnhealthyPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + world_index: int = Field(ge=0) + cause: str = Field(max_length=200) + + +class ScenarioStartedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + scenario_key: str = Field(min_length=1) + world_index: int = Field(ge=0) + scenario_attempt: Literal[1, 2] + + +class ScenarioRetriedPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + scenario_key: str = Field(min_length=1) + from_world: int = Field(ge=0) + to_world: int = Field(ge=0) + + +class LogPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + level: LogLevel + message: str + + +_LOG_TRUNCATION_MARKER = "…[truncated]" + + +def truncate_log_message( + level: str, message: str, *, max_payload_bytes: int = EVENT_PAYLOAD_MAX_BYTES +) -> str: + """The `log` event's own contract rule -- "truncated to fit with a trailing `…[truncated]` + marker" -- unlike the other eight event types, which are hard-rejected when oversized (M8: + `log` is the contract's designated escape hatch for reporting every other permanent failure, so + the one channel meant to report an oversized diagnostic must not itself throw on size). + + Sizing is against `canonical_bytes({"level": level, "message": })`, the exact bytes + `HostedEventDraft`'s own size check measures, so a truncated message is guaranteed to fit + before it ever reaches that check. A no-op when already within budget. + """ + if len(canonical_bytes({"level": level, "message": message})) <= max_payload_bytes: + return message + if len(canonical_bytes({"level": level, "message": _LOG_TRUNCATION_MARKER})) > max_payload_bytes: + raise OutboundError( + "log_payload_budget_too_small", + f"max_payload_bytes={max_payload_bytes} cannot fit even the truncation marker", + ) + lo, hi, best = 0, len(message), "" + while lo <= hi: + mid = (lo + hi) // 2 + candidate = message[:mid] + _LOG_TRUNCATION_MARKER + if len(canonical_bytes({"level": level, "message": candidate})) <= max_payload_bytes: + best = candidate + lo = mid + 1 + else: + hi = mid - 1 + return best + + +class TerminalFailure(BaseModel): + """The terminal event's `failure` shape: `{domain, stage, code, message}` -- a leaner subset of + `job.HarnessFailure` (no `retryable`/`details`), matching §"Event `type` vocabulary" exactly so + a canonicalized terminal payload never carries fields the contract doesn't name.""" + + model_config = ConfigDict(extra="forbid") + + domain: FailureDomain + stage: HarnessStage + code: str + message: str + + +class ScenarioCounts(BaseModel): + model_config = ConfigDict(extra="forbid") + + passed: int = Field(ge=0) + failed: int = Field(ge=0) + errored: int = Field(ge=0) + skipped: int = Field(ge=0) + + +class TerminalPayload(BaseModel): + model_config = ConfigDict(extra="forbid") + + stage: HarnessStage + reason: TerminalReason | None + failure: TerminalFailure | None + scenario_counts: ScenarioCounts + + @model_validator(mode="after") + def _terminal_stage(self) -> "TerminalPayload": + if not self.stage.terminal: + raise ValueError(f"terminal_event_stage_not_terminal: {self.stage.value}") + return self + + +_PAYLOAD_MODELS: dict[OutboundEventType, type[BaseModel]] = { + OutboundEventType.STAGE_CHANGED: StageChangedPayload, + OutboundEventType.PARALLELISM_DEGRADED: ParallelismDegradedPayload, + OutboundEventType.BASELINE_FROZEN: BaselineFrozenPayload, + OutboundEventType.BASELINE_INPUTS_CHANGED: BaselineInputsChangedPayload, + OutboundEventType.WORLD_UNHEALTHY: WorldUnhealthyPayload, + OutboundEventType.SCENARIO_STARTED: ScenarioStartedPayload, + OutboundEventType.SCENARIO_RETRIED: ScenarioRetriedPayload, + OutboundEventType.LOG: LogPayload, + OutboundEventType.TERMINAL: TerminalPayload, +} + + +class HostedEventDraft(BaseModel): + """A Channel 1 event before spool-assigned `sequence`. The caller supplies `digest` itself + (computed via `event_payload_digest`) -- the model then re-derives it and rejects a mismatch, + so a caller can never accidentally spool a record whose embedded digest disagrees with its own + payload bytes. + """ + + model_config = ConfigDict(extra="forbid") + + event_id: str = Field(min_length=1, max_length=64) + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + emitted_at: UtcMillisDatetime + stage: HarnessStage + type: OutboundEventType + payload: dict[str, JsonValue] + digest: str + + @field_serializer("emitted_at") + def _serialize_emitted_at(self, value: datetime) -> str: + return format_rfc3339_millis(value) + + @model_validator(mode="after") + def _validate(self) -> "HostedEventDraft": + # `max_length=64` above counts characters; the contract says "opaque <=64 chars," but the + # platform's column is presumably bytes -- a multi-byte-UTF-8 id could pass the character + # count and still overflow it (N-2). + if len(self.event_id.encode("utf-8")) > 64: + raise ValueError(f"event_id_too_long_in_bytes: {self.event_id!r}") + if not is_valid_digest(self.digest): + raise ValueError(f"event_digest_invalid: {self.digest!r}") + expected = event_payload_digest(self.payload) + if self.digest != expected: + raise ValueError("event_digest_mismatch") + if len(canonical_bytes(self.payload)) > EVENT_PAYLOAD_MAX_BYTES: + raise ValueError(f"event_payload_too_large: {self.event_id}") + + model_cls = _PAYLOAD_MODELS[self.type] + try: + model_cls.model_validate(self.payload) + except ValidationError as exc: + raise ValueError(f"event_payload_invalid: {self.type.value}: {exc}") from exc + + if self.type is OutboundEventType.STAGE_CHANGED and self.payload.get("to") != self.stage.value: + raise ValueError("event_stage_mismatch: stage_changed.to must equal the event's stage") + if self.type is OutboundEventType.TERMINAL and self.payload.get("stage") != self.stage.value: + raise ValueError("event_stage_mismatch: terminal.stage must equal the event's stage") + return self + + +class HostedEvent(HostedEventDraft): + """The full Channel 1 wire object, `sequence` included -- what actually gets spooled and sent. + Distinct from `fi.simulate.runtime.events.CanonicalEvent` (the untouched local-SDK wire).""" + + sequence: int = Field(ge=1) + + +def build_event_record( + *, + event_id: str, + job_id: str, + attempt_id: str, + attempt_number: int, + emitted_at: datetime, + stage: HarnessStage, + type: OutboundEventType, + payload: dict[str, JsonValue], + extra_secret_values: tuple[str, ...] = (), +) -> dict[str, Any]: + """Validate one event's shape and compute its digest, returning a plain dict with no + `sequence` key -- ready for `OutboundSpool.append`, which assigns `sequence` and performs the + one-time serialization. Raises `ValueError` (via pydantic) on any shape violation; callers that + want a typed/coded failure should catch `pydantic.ValidationError` themselves, matching how the + rest of this package surfaces model-layer rejections (`bundle_v2.py`, `job.py`). + + N9: `redact_outbound_text` runs on every free-text field the contract names BEFORE the digest + is computed -- `log.message`, `world_unhealthy.cause`, `terminal.failure.{code,message}`, + `baseline_frozen.baseline_ref` (P8) -- so the embedded digest always matches the redacted bytes + actually spooled and sent, never the unredacted original. `log` events are then truncated to + fit (M8), also before the digest -- redact first, since truncation must size against the final + (redacted) text, not text that would still shrink again once secrets are scrubbed. Every other + event type still hard-rejects when oversized, via `HostedEventDraft`'s own size check. + + P8: `failure.code` is redacted alongside `failure.message` -- both are free `str` fields (the + contract's `code` vocabularies are closed in prose, but nothing enforces that here), and + `baseline_ref` is likewise a free `str` that plausibly carries an OCI/registry reference in the + same `https://@registry/...` shape `redact_outbound_text` already scrubs. + """ + payload = dict(payload) + if type is OutboundEventType.LOG: + level, message = payload.get("level"), payload.get("message") + if isinstance(message, str): + message = redact_outbound_text(message, extra_secret_values) + if isinstance(level, str): + message = truncate_log_message(level, message) + payload["message"] = message + elif type is OutboundEventType.WORLD_UNHEALTHY: + cause = payload.get("cause") + if isinstance(cause, str): + payload["cause"] = redact_outbound_text(cause, extra_secret_values) + elif type is OutboundEventType.BASELINE_FROZEN: + baseline_ref = payload.get("baseline_ref") + if isinstance(baseline_ref, str): + payload["baseline_ref"] = redact_outbound_text(baseline_ref, extra_secret_values) + elif type is OutboundEventType.TERMINAL: + failure = payload.get("failure") + if isinstance(failure, dict): + redacted_failure = dict(failure) + if isinstance(failure.get("code"), str): + redacted_failure["code"] = redact_outbound_text(failure["code"], extra_secret_values) + if isinstance(failure.get("message"), str): + redacted_failure["message"] = redact_outbound_text(failure["message"], extra_secret_values) + payload["failure"] = redacted_failure + digest = event_payload_digest(payload) + draft = HostedEventDraft( + event_id=event_id, + job_id=job_id, + attempt_id=attempt_id, + attempt_number=attempt_number, + emitted_at=emitted_at, + stage=stage, + type=type, + payload=payload, + digest=digest, + ) + return draft.model_dump(mode="json") + + +# ================================================================================================= +# Spool -- durable on-disk queue + monotonic sequence allocator. +# ================================================================================================= + + +@dataclass(frozen=True) +class SpooledRecord: + """One durably-appended record. `body` is the EXACT canonical bytes written to disk -- a P8 + transport client re-sends `body` verbatim on retry rather than re-serializing the decoded + dict, per the contract's "serialize once ... never re-serialize on retry.\"""" + + sequence: int | None + body: bytes + + def decode(self) -> dict[str, Any]: + return json.loads(self.body.decode("utf-8")) + + +class OutboundSpoolError(RuntimeError): + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +def _iter_complete_records( + data: bytes, +) -> tuple[list[tuple[int, bytes, dict[str, Any] | list[Any] | str | int | float | bool | None]], int, int | None]: + """Shared by `_recover` and `records()`/`pending_since_watermark()` -- the ONE place spool + bytes are split into records, so both ever agree on what a "complete record" is. + + Splits on the literal `b"\\n"` byte (N-1: not `bytes.splitlines()`, which also treats `\\r`/ + `\\r\\n` as separators `append` never writes -- canonical JSON never contains a raw newline of + any kind, so `\\n` is the only byte that can legitimately end a line). + + Returns `(records_before_corruption, valid_length, corruption_offset)` (N8): + - `records_before_corruption`: every complete, successfully decoded, non-blank line UP TO the + first corrupt one (or all of them, if none is corrupt), as `(start_offset, raw_line, + decoded_value)`, in file order. + - `corruption_offset`: the byte offset of the first `\\n`-terminated line that failed to parse + as JSON -- genuine corruption (a torn write, by definition, never got its trailing `\\n`, so + this is never that) -- or `None` if no such line was found. Once found, scanning STOPS: never + renumber or trust anything past a corrupt byte (B1). + - `valid_length`: when `corruption_offset is None`, the byte offset immediately after the last + complete line -- where `_recover` truncates away a torn tail. When corruption WAS found, this + equals `corruption_offset` and callers must NOT use it to truncate -- corrupt bytes are left + on disk, never deleted (N8: "never truncates mid-file damage"). + """ + records: list[tuple[int, bytes, Any]] = [] + start = 0 + while True: + newline_index = data.find(b"\n", start) + if newline_index == -1: + return records, start, None + line = data[start:newline_index] + line_end = newline_index + 1 + if line: + try: + decoded = json.loads(line.decode("utf-8")) + except ValueError: + return records, start, start + records.append((start, line, decoded)) + start = line_end + + +class OutboundSpool: + """Durable, crash-safe local queue for one outbound record stream (events, results, or + artifact-manifest state) -- the "fsync-first local spool" the contract requires emission to sit + behind ("Emission is an async flusher over the fsync-first local spool -- it never blocks the + call loop"). `sequenced=True` is for Channel 1 only ("Sequencing: one allocator, one lock, + assigned at spool append, contiguous from 1" -- receipts and the manifest carry no `sequence` + field and use their own idempotency keys instead). + + ONE ALLOCATOR PER STREAM (M6): `OutboundSpool(root, name, ...)` is keyed on + `(resolved_root, name)` -- a second construction for the same key, anywhere in this process, + returns the SAME instance rather than a second independent allocator (see `__new__`); a second + OS PROCESS pointing at the same directory fails loudly instead, via an `fcntl.flock` on + `.spool.lock` held for the life of the owning instance. + + Recovery rule (the contract specifies the sequencing invariant -- contiguous from 1, no gaps or + dupes across a restart -- but not the recovery mechanism; this is the FAIL-SAFE/REVERSIBLE + choice under the stuck-decision rule, surfaced in the P7 report): + + The next sequence number is derived by SCANNING the spool's own JSONL log at startup, never + from an independent counter file. A separate counter file could be durably advanced in a write + that lands, while the record it was allocated for does not (crash between the two writes), + producing a sequence number with no corresponding record -- a permanent, undetectable gap. + Scanning the log makes the durably-written records themselves the only source of truth. The + scan is then reconciled against the durable watermark (M5): `next_sequence = + max(max_sequence_in_log, watermark) + 1` -- the watermark can be AHEAD of the log (the log lost + already-processed records, e.g. via the directory-fsync gap M2 closes) but never behind it, so + taking the max is always safe and never skips a record that was actually spooled. + + A torn last line -- a crash mid-write, since a single `write()` of `body + b"\\n"` is not + guaranteed atomic by POSIX for a regular file -- is detected (the trailing bytes don't end in + `b"\\n"`) and the file is truncated back to the end of the last complete record before any + further append. The next append then reuses that same sequence number rather than skipping it: + a torn write is treated as though it never happened, closing the gap instead of creating one. + This depends on canonical JSON never containing a raw newline byte (control characters are + always escaped by `json.dumps`), which `append` asserts on every write. A COMPLETE line that + still fails to parse is a different, worse fault: genuine corruption degrades the stream to its + readable prefix rather than raising (N8, `is_corrupt`) -- see `_recover`/`records()`. + + Registration (N4): a construction is only added to `_registry` at the END of a successful + `__init__`, under `_registry_lock` -- never a half-built instance. A failed construction (e.g. + `mkdir` EACCES, or `_recover` finding corruption) therefore never poisons the key: it raises + without registering anything, and the NEXT `OutboundSpool(root, name, ...)` call starts a + completely fresh attempt rather than returning (or conflicting with) wreckage. The real mutual- + exclusion primitive across a same-key construction race is `_acquire_process_lock`'s `flock` + (an OS-level device, safe across threads and processes alike) -- the registry dict on top is + only a same-process memoization cache. + """ + + _registry: ClassVar[dict[tuple[Path, str], "OutboundSpool"]] = {} + _registry_lock: ClassVar[RLock] = RLock() + + def __new__(cls, root: str | Path, name: str, *, sequenced: bool) -> "OutboundSpool": + resolved_root = Path(root).expanduser().resolve() + key = (resolved_root, name) + with cls._registry_lock: + existing = cls._registry.get(key) + if existing is not None: + # getattr belt-and-braces (N4): `existing` is only ever registered after a fully + # successful __init__, so `_sequenced` should always be set -- but never trust that + # invariant harder than a defensive read costs. + if getattr(existing, "_sequenced", None) != sequenced: + raise OutboundSpoolError( + "outbound_spool_sequenced_mismatch", + f"{name}: existing instance has sequenced={getattr(existing, '_sequenced', None)}, " + f"requested sequenced={sequenced}", + ) + return existing + return super().__new__(cls) + + def __init__(self, root: str | Path, name: str, *, sequenced: bool) -> None: + if getattr(self, "_initialized", False): + return + resolved_root = Path(root).expanduser().resolve() + key = (resolved_root, name) + with type(self)._registry_lock: + if getattr(self, "_initialized", False): + return + lock_fd: int | None = None + try: + self.root = resolved_root + self.root.mkdir(parents=True, exist_ok=True) + try: + os.chmod(self.root, 0o700) # MIN-10: mkdir's mode is subject to umask + except OSError: + pass + self._name = name + self._sequenced = sequenced + self._path = self.root / f"{name}.spool.jsonl" + self._watermark_path = self.root / f"{name}.spool.watermark.json" + self._lock = RLock() + self._dir_synced = False + self._offset_by_sequence: dict[int, int] = {} + self._next_sequence = 1 if sequenced else None + self._poisoned = False # N12 + self._corrupt_since_offset: int | None = None # N8 + self._forked = False # N25 + self._closed = False # P2 + self._lock_fd = self._acquire_process_lock() + lock_fd = self._lock_fd + self._recover() + except BaseException: + # N4: never leave a half-built instance registered -- it was never added (below), + # so there is nothing to evict; just release whatever this attempt itself opened. + if lock_fd is not None: + try: + os.close(lock_fd) + except OSError: + pass + raise + self._initialized = True + type(self)._registry[key] = self + + @classmethod + def _forget_for_tests(cls, root: str | Path, name: str) -> None: + """Test-only escape hatch: a real process restart naturally gets a fresh, empty registry + (a new interpreter); simulating that WITHIN one process/test needs an explicit evict so the + next `OutboundSpool(root, name, ...)` call re-scans the on-disk log instead of returning the + still-live cached instance. Never called from production code.""" + resolved_root = Path(root).expanduser().resolve() + with cls._registry_lock: + instance = cls._registry.get((resolved_root, name)) + if instance is not None: + instance.close() + + @classmethod + def _clear_registry_for_tests(cls) -> None: + """Broader sibling of `_forget_for_tests`: releases every cached instance's lock fd and + empties the registry. Intended for an autouse test fixture so flock fds don't accumulate + across a whole test session.""" + with cls._registry_lock: + instances = list(cls._registry.values()) + for instance in instances: + instance.close() + + def close(self) -> None: + """N26/P2: releases this instance's process lock and evicts it from the registry, so a + later `OutboundSpool(root, name, ...)` call re-scans the on-disk log instead of reusing this + instance. Idempotent -- safe to call more than once, or on an instance never fully + constructed. + + P2: also sets `_closed`, so THIS instance -- not just the registry slot -- refuses further + mutation. Evicting the registry entry alone left the closed instance itself fully live: a + caller still holding a reference could keep appending with no flock held (the lock fd was + released), and a fresh `OutboundSpool(...)` call for the same key would allocate a second, + independent `_next_sequence` -- two live allocators for one stream, each unaware of the + other, which is exactly the M6 invariant `close()` must not itself reopen.""" + with type(self)._registry_lock: + key = (getattr(self, "root", None), getattr(self, "_name", None)) + if type(self)._registry.get(key) is self: + del type(self)._registry[key] + fd = getattr(self, "_lock_fd", None) + if fd is not None: + try: + os.close(fd) + except OSError: + pass + self._lock_fd = None + self._closed = True + + def _require_writable(self) -> None: + """P2/P7: the single gate `append`, `advance_watermark`, and `_rewrite_retaining` all call + before touching disk -- refuses a closed, forked, or poisoned instance instead of letting it + silently duplicate allocators, advance a parent's watermark from a forked child, or compound + a rollback failure `_truncate_to` already flagged as unrecoverable.""" + if self._closed: + raise OutboundSpoolError( + "outbound_spool_closed", + f"{self._name}: this OutboundSpool was closed; construct a new one for this stream", + ) + if self._forked: + raise OutboundSpoolError( + "outbound_spool_forked", + f"{self._name}: this OutboundSpool was constructed before a fork; construct a " + f"new one in the child process instead of reusing this one", + ) + if self._poisoned: + raise OutboundSpoolError( + "outbound_spool_poisoned", + f"{self._name}: a prior rollback failed and left this spool in an unknown " + f"state; it must not be mutated again", + ) + + def _acquire_process_lock(self) -> int: + """M6: cross-PROCESS protection (the in-process registry above only protects against a + second Python-level instance in this same interpreter). Held for the life of this instance + -- released implicitly when its fd closes (process exit, `close()`, or `_forget_for_tests` + in tests).""" + if fcntl is None: # N30 + raise OutboundSpoolError( + "outbound_spool_platform_unsupported", + f"{self._name}: fcntl (POSIX file locking) is unavailable on this platform", + ) + lock_path = self.root / f"{self._name}.spool.lock" + fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + os.close(fd) + raise OutboundSpoolError( + "outbound_spool_locked", f"{self._name}: already locked by another process" + ) from exc + return fd + + def _fsync_dir(self) -> None: + fd = os.open(str(self.root), os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + def _report_corruption(self, offset: int) -> None: + """N8: the client-visible half of "degrade, don't wedge" -- `is_corrupt`/`corruption_offset` + stay true/set for the life of this instance once discovered, but the loud `logger.error` + fires only the FIRST time (repeated reads of an already-known-corrupt spool would otherwise + spam the log every flush cycle).""" + with self._lock: + already_reported = self._corrupt_since_offset is not None + self._corrupt_since_offset = offset + if not already_reported: + logger.error( + "%s: spool corrupt at byte offset %d -- the file is left untouched; only records " + "before that offset are trusted. Reading degrades to the readable prefix rather " + "than raising -- this is reported once per process, not per read.", + self._name, offset, + ) + + @property + def is_corrupt(self) -> bool: + """N8: set once `_recover` or a later read finds a genuinely corrupt (not merely torn) + record. A caller can turn this into a `log`-kind event; nothing in this class does that + itself (no scenario/attempt context here to build one).""" + return self._corrupt_since_offset is not None + + @property + def corruption_offset(self) -> int | None: + return self._corrupt_since_offset + + def _recover(self) -> None: + # NOTE: the sequenced branch below must run even when the log is missing or empty -- an + # M2-style lost file (durable watermark, vanished log) is exactly the case M5 needs to + # reconcile against; an early return here would skip that reconciliation entirely and + # silently reset the allocator to 1. + records: list[tuple[int, bytes, Any]] = [] + if self._path.exists(): + data = self._path.read_bytes() + if data: + records, valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: + # N8: never truncate mid-file damage -- leave the bytes exactly as they are, + # trust only what came before, and let construction succeed anyway (B1 already + # forbids renumbering past it; the OTHER extreme -- raising here -- would + # discard every future emit, including the terminal event, forever). + self._report_corruption(corruption_offset) + elif valid_length < len(data): + with self._path.open("r+b") as stream: + stream.truncate(valid_length) + stream.flush() + os.fsync(stream.fileno()) # MIN-11: durable, not left as a crash window + if self._sequenced: + max_sequence = 0 + offsets: dict[int, int] = {} + for offset, _line, record in records: + if isinstance(record, dict): + sequence = record.get("sequence") + if isinstance(sequence, int): + offsets[sequence] = offset + if sequence > max_sequence: + max_sequence = sequence + self._offset_by_sequence = offsets + watermark = self.watermark() + if watermark > max_sequence: + logger.warning( + "%s: watermark (%s) is ahead of the highest sequence found in the spool (%s) " + "-- the log lost records the platform already processed; seeding " + "next_sequence from the watermark so newly allocated sequences don't collide " + "with ones the platform already closed", + self._name, watermark, max_sequence, + ) + self._next_sequence = max(max_sequence, watermark) + 1 + + def _truncate_to(self, size: int) -> None: + """B1: a true no-op on a failed append. `_next_sequence` is only advanced AFTER a + successful write, so the retry reuses the same sequence number -- this makes sure it reuses + clean ground too, instead of appending immediately after torn bytes with no `\\n` between + them (which would merge into one unparseable line the rest of this class can't recover + from).""" + try: + with self._path.open("r+b") as stream: + stream.truncate(size) + stream.flush() + os.fsync(stream.fileno()) + except OSError as exc: + # N12: the write may have failed before the file even existed (nothing to truncate, + # harmless) OR the rollback itself failed on an existing torn write -- in the latter + # case B1's "a failed append is a true no-op" no longer holds, so poison this spool + # rather than let a future append silently merge into the torn bytes. + self._poisoned = True + logger.error( + "%s: rollback of a failed append could not truncate the spool back to %d bytes " + "(%s) -- the file may now carry torn bytes; poisoning this spool so a caller sees " + "a typed error instead of a future append compounding the corruption", + self._name, size, exc, + ) + + def append(self, record: dict[str, Any]) -> SpooledRecord: + with self._lock: + self._require_writable() + if self._sequenced: + assigned = self._next_sequence + record = {**record, "sequence": assigned} + elif "sequence" in record: + raise OutboundSpoolError( + "outbound_spool_caller_supplied_sequence", + f"{self._name} spool does not assign sequence numbers; caller must not pass one", + ) + body = canonical_bytes(record) + if b"\n" in body: + # Framing invariant `_iter_complete_records`/`_recover` depend on: canonical JSON + # never contains a raw newline (json.dumps escapes control characters inside + # strings), so this would only fire on a value this module's own canonicalization + # contract disallows. + raise OutboundSpoolError( + "outbound_spool_record_unframable", f"{self._name}: record contains a raw newline" + ) + existed_before = self._path.exists() + size_before = self._path.stat().st_size if existed_before else 0 + try: + with self._path.open("ab") as stream: + stream.write(body) + stream.write(b"\n") + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + self._truncate_to(size_before) + raise + if not existed_before and not self._dir_synced: + # M2: the directory entry for a brand-new file isn't durable just because the + # file's own data is -- fsync it once (not per append; existing files' entries were + # already synced by whichever append first created them). + self._fsync_dir() + self._dir_synced = True + if self._sequenced: + self._offset_by_sequence[assigned] = size_before + self._next_sequence = assigned + 1 + return SpooledRecord(sequence=assigned, body=body) + return SpooledRecord(sequence=None, body=body) + + def records(self) -> list[SpooledRecord]: + """Reads the WHOLE log. The read itself happens OUTSIDE `self._lock` (M4): only the size + snapshot that bounds it is taken under the lock, so the flusher's (potentially large) read + never blocks `append`, the call loop's write path, for its duration. Safe because `append` + only ever grows the file -- a read bounded to a size captured a moment earlier can only be + stale, never torn. (`compact_through`/`drop` DO shrink the file and take the lock for their + entire duration; this module assumes the flusher serializes its own reads against its own + compactions rather than running them from two different threads.) + """ + with self._lock: + if not self._path.exists(): + return [] + size = self._path.stat().st_size + with self._path.open("rb") as stream: + data = stream.read(size) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: # N8: degrade to the readable prefix, never raise here + self._report_corruption(corruption_offset) + out: list[SpooledRecord] = [] + for _offset, line, decoded in parsed: + sequence = decoded.get("sequence") if self._sequenced and isinstance(decoded, dict) else None + out.append(SpooledRecord(sequence=sequence, body=line)) + return out + + def records_after(self, sequence: int) -> list[SpooledRecord]: + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + return [item for item in self.records() if (item.sequence or 0) > sequence] + + @property + def next_sequence(self) -> int: + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + assert self._next_sequence is not None + return self._next_sequence + + def watermark(self) -> int: + """The highest-processed sequence acknowledged so far ("the watermark is highest-processed + -- accepted AND rejected sequences both advance it"). Durable across a restart via a + fsync'd-temp-then-rename-then-fsync'd-directory write (M1) -- a corrupt or unreadable + watermark file DEGRADES TO 0 with a loud diagnostic rather than raising and wedging the + spool: re-sending already-acked events is safe (at-least-once delivery + platform-side + dedupe on `event_id`), while a permanently unreadable outbound channel is not. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + if not self._watermark_path.exists(): + return 0 + try: + raw = json.loads(self._watermark_path.read_text(encoding="utf-8")) + return int(raw["acked_through_sequence"]) + except (OSError, ValueError, KeyError, TypeError) as exc: + logger.warning( + "%s: watermark file is corrupt or unreadable (%s) -- degrading to 0. Re-sending " + "already-acked events is safe (at-least-once + dedupe on event_id); wedging the " + "spool permanently is not.", + self._name, exc, + ) + return 0 + + def advance_watermark(self, sequence: int) -> None: + """v1.3: `acked_through_sequence` is untrusted platform input. A value outside + `[current_watermark, next_sequence)` is rejected locally with a typed error and the + watermark is left exactly as it was -- "a malformed ack must not be able to discard + pending records" (M7). `sequence == current_watermark` is a legitimate no-op, not an error + (repeating the same ack, or a same-valued out-of-order response). + + P7: this is the one operation that DESTROYS delivery state (it durably advances what a + future `pending_since_watermark()` will ever return again) -- a forked child or a poisoned + instance advancing it would silently orphan every pending record below the new value, the + N1 outcome by a different route. `_require_writable()` guards it for that reason even + though nothing here writes to the JSONL log itself. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + with self._lock: + self._require_writable() + current = self.watermark() + if sequence < current or sequence >= self._next_sequence: + raise OutboundSpoolError( + "outbound_spool_watermark_out_of_range", + f"{self._name}: acked_through_sequence={sequence} outside the trusted range " + f"[{current}, {self._next_sequence}) -- untrusted platform input, watermark " + f"left unchanged", + ) + if sequence == current: + return + temporary = self.root / f"{self._name}.spool.watermark.tmp.{os.getpid()}.{uuid.uuid4().hex}" + with temporary.open("wb") as stream: + stream.write( + json.dumps({"acked_through_sequence": sequence}, separators=(",", ":")).encode( + "utf-8" + ) + ) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self._watermark_path) + self._fsync_dir() + + def pending_since_watermark(self) -> list[SpooledRecord]: + """Convenience for "the guest advances its spool cursor through the watermark": every + spooled record not yet acknowledged, in sequence order. Uses the offset `append` recorded + for the first pending sequence to SEEK directly there (M4) instead of re-reading and + re-parsing the whole log on every flush cycle; falls back to the generic full scan when no + cached offset exists yet (e.g. a fresh recovery whose watermark sits past every record this + process itself has written an offset for). + + N13: the drained steady state (nothing pending -- what a polling flusher sees most cycles) + is checked first and returns `[]` with NO file IO at all: `watermark + 1 == next_sequence` + means every allocated sequence has already been acknowledged, so there is nothing on disk + to seek to regardless of what `_offset_by_sequence` does or doesn't have cached. + """ + watermark = self.watermark() + with self._lock: + if watermark + 1 == self._next_sequence: + return [] + offset = self._offset_by_sequence.get(watermark + 1) + if offset is None: + return self.records_after(watermark) + with self._lock: + if not self._path.exists(): + return [] + size = self._path.stat().st_size + if offset >= size: + return [] + with self._path.open("rb") as stream: + stream.seek(offset) + data = stream.read(size - offset) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: # N8: absolute offset -- `data` starts at `offset` + self._report_corruption(offset + corruption_offset) + return [ + SpooledRecord(sequence=decoded.get("sequence") if isinstance(decoded, dict) else None, body=line) + for _offset, line, decoded in parsed + ] + + def _rewrite_retaining(self, keep: Callable[[Any], bool]) -> None: + """Shared by `compact_through` and `drop_many`: rewrites the log keeping only records + `keep` accepts, via the same write-fsync/atomic-replace/fsync-directory durability shape + `append`/`advance_watermark` use -- a crash mid-rewrite leaves either the old file intact + or the new one complete, never a torn hybrid. + + P1: if the log is already corrupt (N8), this REFUSES instead of rewriting. A rewrite always + replaces the file from what it read, and reading stops at the corruption offset -- so + rewriting on a corrupt spool would not merely skip the corrupt bytes, it would silently + destroy every intact record PAST them too, including ones this process itself appended + after recovery that have never been sent (the terminal event, in the worst case). That + directly defeats N8's "the readable prefix stays usable, delivery keeps working" posture + the moment the first drop/compact happens. Refusing costs only unbounded disk growth until + the attempt ends (`compact_through`'s whole job) or a rejected record staying spooled but + never re-emitted anyway, since it's at or below the watermark (`drop_many`'s whole job) -- + both strictly better than deleting undelivered records. + """ + with self._lock: + self._require_writable() + if not self._path.exists(): + return + size = self._path.stat().st_size + with self._path.open("rb") as stream: + data = stream.read(size) + parsed, _valid_length, corruption_offset = _iter_complete_records(data) + if corruption_offset is not None: + self._report_corruption(corruption_offset) + logger.error( + "%s: refusing to compact/drop on a corrupt spool -- a rewrite replaces the file " + "from what it read, and reading stops at byte %d, so every record past that " + "offset (including not-yet-delivered ones) would be destroyed. The log is left " + "intact and grows unbounded until the attempt ends; that is the fail-safe half " + "of degrade-not-wedge.", + self._name, corruption_offset, + ) + return + temporary = self.root / f"{self._name}.spool.jsonl.tmp.{os.getpid()}.{uuid.uuid4().hex}" + offsets: dict[int, int] = {} + offset = 0 + with temporary.open("wb") as stream: + for _old_offset, line, decoded in parsed: + if not keep(decoded): + continue + if self._sequenced and isinstance(decoded, dict) and isinstance(decoded.get("sequence"), int): + offsets[decoded["sequence"]] = offset + stream.write(line) + stream.write(b"\n") + offset += len(line) + 1 + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, self._path) + self._fsync_dir() + if self._sequenced: + self._offset_by_sequence = offsets + + def compact_through(self, sequence: int) -> None: + """M4: physically drops every durably-acked record (`sequence <= min(sequence, + watermark())`) from the on-disk log, bounding its growth for a long `running` stage. The + allocator's `next_sequence` is unaffected -- it is only ever derived from the log at + `_recover` time, and recovery's own `max(max_sequence, watermark)` rule (M5) already + tolerates a log whose historical records were compacted away, since none of them can be + the true maximum (compaction only ever removes sequences at or below the watermark, and + the watermark is always <= every pending, uncompacted sequence). + + Clamped to the current watermark regardless of what the caller passes -- compacting past + an event the platform hasn't actually processed yet would be irreversible data loss, and + this module's posture throughout is fail-safe over trusting the caller. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + effective = min(sequence, self.watermark()) + self._rewrite_retaining( + lambda decoded: not ( + isinstance(decoded, dict) + and isinstance(decoded.get("sequence"), int) + and decoded["sequence"] <= effective + ) + ) + + def drop_many(self, sequences: Collection[int]) -> None: + """The contract's rejected-event mechanism: "a rejected event is dropped from the spool ... + it is never re-emitted" (M5). PURE physical removal (N1) -- every sequence in `sequences` + is deleted from the on-disk log in ONE rewrite pass (N14: a batch of 100 rejections is one + `_rewrite_retaining` call, not 100), and the watermark is left untouched. + + N1: an earlier version had `drop` also advance the watermark to `sequence`, on the theory + that "a rejected event closes its sequence." That let an UNTRUSTED `rejected[].sequence` + from the platform silently orphan every pending record below it, bypassing + `advance_watermark`'s own M7 clamp entirely -- the clamp only guards `acked_through_sequence` + callers, and `drop` skipped straight past it. The batch-level + `advance_watermark(acked_through_sequence)` a caller performs separately is the ONE place + the watermark ever moves; it already covers every rejected sequence under a conformant + platform (rejections advance the watermark by contract), and under a non-conformant one + M7's clamp is then the single, correct chokepoint -- this method has no clamp of its own to + bypass. + + Writing the record's payload to the artifact spool as a `log` kind (the other half of the + contract's drop rule) is NOT this method's job -- that hand-off needs scenario/attempt + context and an `ArtifactsClient` this layer doesn't own; it is P9/P10 wiring, documented + here as the seam rather than guessed at. + """ + if not self._sequenced: + raise OutboundSpoolError("outbound_spool_unsequenced", self._name) + sequence_set = set(sequences) + if not sequence_set: + return + self._rewrite_retaining( + lambda decoded: not (isinstance(decoded, dict) and decoded.get("sequence") in sequence_set) + ) + + def drop(self, sequence: int) -> None: + """Single-sequence convenience wrapper over `drop_many` -- see its docstring for why this + no longer touches the watermark.""" + self.drop_many((sequence,)) + + +def _poison_after_fork() -> None: + """N25: `os.fork()` inherits both `OutboundSpool._registry` (with a live `_next_sequence`) and + every instance's flock fd (the SAME open file description, so the lock is merely shared, not + contended, across parent and child) -- without this, parent and child would allocate identical + sequence numbers with no complaint. Marks every currently-registered instance so its next + mutating call raises instead. Not reachable via `subprocess` (fork+exec resets memory); this + guards a bare `os.fork()` specifically.""" + with OutboundSpool._registry_lock: + for instance in OutboundSpool._registry.values(): + instance._forked = True + + +if hasattr(os, "register_at_fork"): # POSIX-only, like fcntl (N30) + # P7: `before=`/`after_in_parent=` pair the registry lock around the fork itself -- without + # this, a fork occurring while some OTHER thread holds `_registry_lock` hands the child a + # locked RLock owned by a thread that no longer exists there, and `_poison_after_fork`'s own + # `with OutboundSpool._registry_lock:` deadlocks at the fork point instead of poisoning + # anything. Acquiring on `before` guarantees the FORKING thread itself owns the lock at fork + # time, so the child's single surviving thread already owns it too -- `_poison_after_fork`'s + # acquire becomes a safe reentrant no-op there, and `after_in_parent` restores normal locking + # in the parent. + os.register_at_fork( + before=OutboundSpool._registry_lock.acquire, + after_in_parent=OutboundSpool._registry_lock.release, + after_in_child=_poison_after_fork, + ) + + +# ================================================================================================= +# Transport -- the HTTP boundary every channel client speaks through, and its production impl. +# ================================================================================================= + + +@dataclass(frozen=True) +class TransportResponse: + status_code: int + body: dict[str, Any] | None + headers: dict[str, str] + + +class TransportError(RuntimeError): + """Raised by a `Transport.request` implementation when no HTTP response was ever received + (connection refused, DNS failure, timeout, ...). `classify_response` treats this identically + to an unreachable 5xx -- the guest cannot distinguish "server errored" from "server unreachable" + and the contract's retry policy doesn't ask it to.""" + + +class Transport(Protocol): + """The seam every channel client is built against, so the fake-platform tests exercise the + exact same code path production traffic does -- only what sits behind this protocol differs. + """ + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Iterator[bytes] | None = None, + timeout: float = 30.0, + ) -> TransportResponse: ... + + +class RequestsTransport: + """Production `Transport`: a thin `requests.Session` wrapper. Every network-layer failure + (`requests.RequestException`, which covers connection errors, timeouts, and retries `requests` + itself doesn't handle) is normalized to `TransportError` so `classify_response` never needs to + know which HTTP library is underneath.""" + + def __init__(self, *, session: requests.Session | None = None) -> None: + self._session = session or requests.Session() + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Iterator[bytes] | None = None, + timeout: float = 30.0, + ) -> TransportResponse: + try: + response = self._session.request( + method, url, headers=headers, json=json_body, data=data, timeout=timeout + ) + except requests.RequestException as exc: + raise TransportError(str(exc)) from exc + try: + body = response.json() if response.content else None + except ValueError: + body = None + return TransportResponse( + status_code=response.status_code, body=body, headers=dict(response.headers) + ) + + +def _iter_chunks(data: bytes, chunk_size: int) -> Iterator[bytes]: + """§3a: "Uploads >64 MB use chunked transfer." A fresh generator is built per send attempt + (never reused across a retry) -- a generator is single-use, and reusing an exhausted one would + silently upload an empty body on the second attempt.""" + for start in range(0, len(data), chunk_size): + yield data[start : start + chunk_size] + + +def _parse_retry_after(headers: Mapping[str, str] | None) -> float | None: + """"429 -> honor `Retry-After`." Only the delta-seconds form is parsed (the integer count of + seconds to wait) -- the contract never mentions the alternative HTTP-date form and every + platform emitter in this ecosystem is expected to send the simple form; an unparseable value is + treated as absent so the caller falls back to the computed backoff rather than crashing. + + N5: a negative value is ALSO treated as absent -- `time.sleep(-5)` raises `ValueError`, and a + server sending a negative `Retry-After` is malformed input this module owes no obedience to. + The upper clamp (`[0, retry_policy.max_backoff_seconds]`) needs the policy, which isn't + available here -- `_perform_with_retry` applies that half. + + P6: header-name lookup is case-INsensitive (RFC 9110 §5.1 -- field names are case-insensitive). + `RequestsTransport` builds `dict(response.headers)` from `requests`' own `CaseInsensitiveDict`, + which drops the case-insensitivity and preserves whatever casing the server actually sent -- a + plain `.get("Retry-After")` would miss `RETRY-AFTER`/`Retry-after` and silently fall back to + computed backoff instead of honoring the server's wait. + """ + if not headers: + return None + value = next((v for k, v in headers.items() if k.lower() == "retry-after"), None) + if value is None: + return None + try: + parsed = float(value) + except ValueError: + return None + return parsed if parsed >= 0 else None + + +# ================================================================================================= +# Error map -- the contract's closed status-code vocabulary ("Error responses" / "Failure semantics +# summary"), and the shared retry engine every channel client drives it through. +# ================================================================================================= + + +class ChannelOutcome(str, Enum): + """Every way one outbound HTTP attempt can resolve, per the contract's failure table. Not a + contract vocabulary itself (the wire only ever carries a status code + `{error, message, + retryable}`) -- this is this module's own closed classification of that table, the thing + `classify_response` computes and every client branches on.""" + + DELIVERED = "delivered" + RETRYABLE = "retryable" + FENCED = "fenced" + PERMANENT_ITEM = "permanent_item" + CHANNEL_FAILED = "channel_failed" + BUDGET_EXCEEDED = "budget_exceeded" + + +@dataclass(frozen=True) +class ChannelError: + outcome: ChannelOutcome + domain: FailureDomain | None + code: str + message: str + retry_after_seconds: float | None = None + # N7: the raw status this was classified from (`None` for a `TransportError`/no-response + # outcome) -- carried so a caller can recognize a specific status (413, for the events-batch + # halving retry) without `ChannelOutcome`/`code` alone being expressive enough for that. + status_code: int | None = None + + +class HostedFencedError(RuntimeError): + """401 (expired) / 403 (fence, scope, mismatch): "stop emitting, exit code 3 ... never an infra + retry." Raised by the shared retry engine and never retried -- the entrypoint (outside this + module) is the one that translates this into the process exit code.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class HostedChannelFailedError(RuntimeError): + """404, retried 3x per the contract, still 404: "finalize `platform_sync`." Raised by the + shared retry engine once `classify_response` reaches the third 404 attempt.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class HostedAttemptSupersededError(RuntimeError): + """N22: `409 attempt_superseded` folded into the `ChannelState` latch below -- "a fenced + attempt's in-flight requests cannot land after registration of its successor" is a fence in + substance, even though the ONE request that received it is still correctly classified + `PERMANENT_ITEM` (contract-correct: 409 is item-level, not fence-level). Only raised by + `ChannelState.check()` on a LATER call, once a prior call has already seen this code -- the + call that actually observed the 409 still returns its normal item-level result.""" + + def __init__(self, error: ChannelError) -> None: + self.error = error + super().__init__(f"{error.code}: {error.message}") + + +class ChannelState: + """N10: shared "stop emitting" latch across the three channel clients for one attempt -- a + fence (401/403) or an exhausted channel (404x3) on ANY channel must stop ALL of them, since + the token/fence is per-ATTEMPT, not per-channel ("stop emitting ... never an infra retry"). + Also carries the N22 attempt-supersession latch (409 `attempt_superseded`). + + Construct ONE `ChannelState` per attempt and pass it to `EventsClient`/`ResultsClient`/ + `ArtifactsClient` alike (each defaults to a private one if not given, which only latches + itself -- correct for a single-channel caller, but callers driving more than one channel for + the same attempt MUST share one instance to get the cross-channel guarantee this class exists + for). Once latched, `check()` raises the SAME error on every subsequent call, from any client + sharing this state, without ever touching the transport. + """ + + def __init__(self) -> None: + self._error: HostedFencedError | HostedChannelFailedError | HostedAttemptSupersededError | None = None + self._lock = RLock() + + def check(self) -> None: + with self._lock: + error = self._error + if error is not None: + raise error + + def latch( + self, exc: "HostedFencedError | HostedChannelFailedError | HostedAttemptSupersededError" + ) -> None: + with self._lock: + if self._error is None: + self._error = exc + + +def classify_response( + status_code: int | None, + body: dict[str, Any] | None, + *, + attempt: int, + retry_after_seconds: float | None = None, +) -> ChannelError | None: + """The single call site every channel client classifies a transport outcome through. Returns + `None` for a delivered (2xx) response, a `ChannelError` for everything else. `status_code=None` + means `TransportError` (no response was ever received) -- classified exactly like an + unreachable 5xx. For the 404 branch specifically, `attempt` must be the count of 404 RESPONSES + seen so far in this call (not the overall attempt number) -- `_perform_with_retry` tracks that + counter separately (N6) so an interleaved 5xx never shortens the 404 budget; every other branch + ignores `attempt` entirely. + + N28: the error body's `retryable` field is deliberately never read here -- classification is + status-keyed throughout this module (every branch below is keyed on `status_code` alone), which + is defensible per the contract's own closed, status-code-driven failure table; if the platform + ever marks an unexpected status `retryable` the guest disagrees silently, a known, accepted gap. + + Every branch below is transcribed directly from the contract's "Error responses" paragraph and + "Failure semantics summary" table -- see `outbound-channels.md` v1.3 for the prose this mirrors. + """ + if status_code is not None and 200 <= status_code < 300: + return None + + error_code = (body or {}).get("error") if body else None + message = (body or {}).get("message", "") if body else "" + + if status_code is None: + return ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + error_code or "network_error", + message or "transport failure: no response received", + status_code=None, + ) + if status_code in (401, 403): + return ChannelError(ChannelOutcome.FENCED, None, error_code or "fenced", message, status_code=status_code) + if status_code == 404: + # N29: PLATFORM_SYNC on every 404 attempt, not just the third -- a 404 is never a + # connectivity fault under §4.6; only the third attempt's outcome is ever surfaced to a + # caller, but the domain should not silently disagree across attempts 1-2 vs 3. + domain = FailureDomain.PLATFORM_SYNC + if attempt < 3: + return ChannelError(ChannelOutcome.RETRYABLE, domain, error_code or "not_found", message, status_code=404) + return ChannelError( + ChannelOutcome.CHANNEL_FAILED, domain, error_code or "not_found", message, status_code=404 + ) + if status_code == 413: + # N7: 413 is Channel 3's artifact-budget code specifically -- only classify it + # BUDGET_EXCEEDED when the body actually says so; a 413 on any other channel (e.g. an + # events batch that simply exceeded the platform's ingress size limit) falls through to + # the unlisted-4xx catch-all below instead of being mislabeled as a budget condition that + # channel doesn't have. + if error_code == "artifact_budget_exceeded": + return ChannelError( + ChannelOutcome.BUDGET_EXCEEDED, None, error_code, message, status_code=413 + ) + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or "http_413", message, status_code=413 + ) + if status_code == 429: + return ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + error_code or "rate_limited", + message, + retry_after_seconds=retry_after_seconds, + status_code=429, + ) + if status_code in (400, 409, 422): + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + if 500 <= status_code < 600: + return ChannelError( + ChannelOutcome.RETRYABLE, FailureDomain.CONNECTIVITY, error_code or "server_error", message, + status_code=status_code, + ) + if 400 <= status_code < 500: + # "Catch-all: any unlisted 4xx is permanent for that item." + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + # N27: no other status family is contractual -- notably a 3xx, which should never occur (every + # endpoint ends in "/" precisely so Django's POST-redirect problem never arises). Treat as + # permanent rather than retrying an endpoint misconfiguration `max_attempts` times before + # giving up anyway. + return ChannelError( + ChannelOutcome.PERMANENT_ITEM, None, error_code or f"http_{status_code}", message, status_code=status_code + ) + + +def compute_backoff_seconds( + attempt: int, + *, + initial_backoff_seconds: float, + max_backoff_seconds: float, + rng: Callable[[], float] = random.random, +) -> float: + """"retry with backoff (base `retry.initial_backoff_seconds`, cap `retry.max_backoff_seconds`, + full jitter)" -- `uniform(0, min(cap, base * 2**(attempt-1)))`. `attempt` is the 1-based attempt + that just failed. `rng` is injectable so callers (and tests) can get a deterministic value. + """ + ceiling = min(max_backoff_seconds, initial_backoff_seconds * (2 ** max(0, attempt - 1))) + return rng() * ceiling + + +@dataclass(frozen=True) +class RetryPolicy: + """Backoff parameters, shared by all three channel clients. Field names mirror + `job.HarnessRetryPolicy` deliberately (the contract states these come from the job's own + `retry.initial_backoff_seconds`/`retry.max_backoff_seconds`), but this module does not import + that class -- a client only needs two floats, and importing the full job-retry model (with its + `retryable_domains` field, which governs WHOLE-JOB attempt retries, a distinct concept from a + single outbound delivery's backoff) would be a coupling this module doesn't need. + + `max_attempts` bounds one `_perform_with_retry` call's own retry loop for the classes the + contract leaves unbounded (network/5xx/429 -- "spool + backoff", no stated attempt ceiling). + STUCK DECISION (fail-safe/reversible, contract silent): capped at a generous default (8) rather + than looped forever, because durability already lives in the spool/idempotent-wire-design, not + in one blocking call -- a caller that wants to keep trying simply invokes the client method + again later (`EventsClient.flush()` is designed to be called repeatedly for exactly this + reason). Must stay >= 3 for the 404 rule to ever reach its own `CHANNEL_FAILED` transition + within a single call; the default comfortably clears that. + """ + + initial_backoff_seconds: float = 1.0 + max_backoff_seconds: float = 15.0 + max_attempts: int = 8 + + +def _perform_with_retry( + perform: Callable[[int], TransportResponse], + *, + retry_policy: RetryPolicy, + sleep: Callable[[float], None], + rng: Callable[[], float] = random.random, + deadline: float | None = None, + now: Callable[[], float] = time.monotonic, +) -> tuple[TransportResponse | None, ChannelError | None]: + """The one retry/backoff engine all three channel clients drive their single HTTP call through. + `perform(attempt)` makes ONE attempt (1-based); this loops it, classifies each outcome via + `classify_response`, and: + + - returns `(response, None)` once delivered; + - raises `HostedFencedError` immediately on 401/403 (never retried, by contract); + - raises `HostedChannelFailedError` once a 404 reaches its third attempt; + - returns `(response_or_None, error)` for every other terminal outcome (`PERMANENT_ITEM`, + `BUDGET_EXCEEDED`) without retrying -- "deterministic rejections are never retried"; + - otherwise (`RETRYABLE`) sleeps -- honoring a server `Retry-After` over the computed backoff + when present -- and tries again, up to `retry_policy.max_attempts`. + + N5/P5: `deadline` (a `time.monotonic()` value, typically the adapter's flush-window end) bounds + ATTEMPT SCHEDULING AND SLEEPS ONLY -- no new attempt starts once `now() >= deadline`, and every + sleep (computed backoff OR a server `Retry-After`) is clamped to whatever budget remains. It + does NOT bound an attempt's own in-flight request: `perform`'s transport call carries its own, + separate `timeout` (the caller's second knob -- `Transport.request`'s `timeout` parameter, + unrelated to `deadline`), and nothing here clamps that value against the remaining deadline + budget. A caller sizing `deadline` against a hard wall-clock guarantee for the whole call is + sizing against a guarantee this function does not provide; only "no new attempt starts, and no + sleep runs, once the window is gone" is guaranteed. (P5: widening `perform` to accept a clamped + per-attempt timeout was considered and is the more complete fix, but at least one caller outside + this module -- `hosted_entrypoint.py`'s `ScenariosClient._post`, which calls this function + directly with its own single-argument `perform` closure -- is out of this fix's scope, so + changing the call signature here would silently break that caller instead of fixing it. This + docstring correction is the floor the round-3 review named for exactly that situation.) + + N6: 404 retries are counted SEPARATELY from the overall attempt number (`not_found_attempts`), + so an interleaved 5xx (e.g. `503, 503, 404, 404, 404`) does not shorten the 404 budget -- only + three OBSERVED 404s reach `classify_response`'s `CHANNEL_FAILED` transition, regardless of what + else happened in between. + """ + attempt = 0 + not_found_attempts = 0 + while True: + attempt += 1 + if deadline is not None and now() >= deadline: + return None, ChannelError( + ChannelOutcome.RETRYABLE, + FailureDomain.CONNECTIVITY, + "deadline_exceeded", + "the flush-window deadline elapsed before this attempt could be made", + ) + try: + response = perform(attempt) + except TransportError: + error = classify_response(None, None, attempt=attempt) + response = None + else: + status = response.status_code + if status == 404: + not_found_attempts += 1 + error = classify_response( + status, + response.body, + attempt=not_found_attempts if status == 404 else attempt, + retry_after_seconds=_parse_retry_after(response.headers), + ) + if error is None: + return response, None + if error.outcome is ChannelOutcome.FENCED: + raise HostedFencedError(error) + if error.outcome is ChannelOutcome.CHANNEL_FAILED: + raise HostedChannelFailedError(error) + if error.outcome is not ChannelOutcome.RETRYABLE: + return response, error + if attempt >= retry_policy.max_attempts: + return response, error + delay = error.retry_after_seconds + if delay is not None: + delay = min(delay, retry_policy.max_backoff_seconds) # N5: clamp Retry-After + else: + delay = compute_backoff_seconds( + attempt, + initial_backoff_seconds=retry_policy.initial_backoff_seconds, + max_backoff_seconds=retry_policy.max_backoff_seconds, + rng=rng, + ) + if deadline is not None: + remaining = deadline - now() + if remaining <= 0: + return response, error + delay = min(delay, remaining) + sleep(delay) + + +# ================================================================================================= +# Channel 1 -- Events client. Batches spooled events, advances the watermark only on confirmed +# delivery. +# ================================================================================================= + + +@dataclass(frozen=True) +class EventsFlushResult: + delivered_count: int + acked_through_sequence: int | None + rejected: list[dict[str, Any]] + error: ChannelError | None + # v1.3 (M7): set when the platform's `acked_through_sequence` fell outside the spool's trusted + # range and was ignored rather than trusted -- `error` stays `None` because the HTTP delivery + # itself succeeded; only the ack body was untrustworthy. + ack_out_of_range: bool = False + # N2/N3: set when a 2xx response's `acked_through_sequence` was missing, `null`, or not an + # int -- a protocol violation distinct from "present but out of range" above. `error` stays + # `None` for the same reason: the HTTP delivery itself succeeded. + ack_missing: bool = False + # P9: the SpooledRecord bodies dropped this call (a subset of `batch`, keyed by `rejected`), + # captured BEFORE `drop_many` removes them from disk. The contract requires a rejected event's + # payload be "written to the artifact spool (`log` kind)" -- without this, a caller has no way + # to recover that payload at all once `flush()` returns, since the cap applied inside `flush()` + # makes it impossible to reliably re-derive which spooled records were even in this batch. + dropped_records: list[SpooledRecord] = field(default_factory=list) + + +_EVENTS_BATCH_PREFIX = b'{"schema_version":"' + EVENT_SCHEMA_VERSION.encode("utf-8") + b'","events":[' +_EVENTS_BATCH_SUFFIX = b"]}" + + +def _encode_events_batch(records: list[SpooledRecord]) -> bytes: + """N19: the contract says "serialize once, spool the bytes, re-send verbatim; never + re-serialize on retry" -- for the events BATCH ENVELOPE itself, not just each event inside it. + Handing a decoded dict to `json_body=` (the previous shape) let `requests` re-serialize the + envelope with its own settings on every send; this instead concatenates the exact bytes + `OutboundSpool.append` already wrote for each event, closing the deviation rather than merely + documenting it. Safe because `EVENT_SCHEMA_VERSION` is a fixed ASCII constant with no bytes + needing escape.""" + return _EVENTS_BATCH_PREFIX + b",".join(record.body for record in records) + _EVENTS_BATCH_SUFFIX + + +class EventsClient: + """Delivers `OutboundSpool`-backed Channel 1 events to `endpoints.events`. One `flush()` call + sends one batch (<= `EVENTS_MAX_BATCH` events, <= `EVENTS_MAX_BATCH_BYTES`) of everything + spooled since the last confirmed watermark, in spool order + (`OutboundSpool.pending_since_watermark`, which reads the log in append/sequence order -- the + platform's own ordering rule, "`(attempt_number, sequence)`", so a single-attempt process + satisfies it for free). Re-sends spooled bytes verbatim (N19, `_encode_events_batch`) -- never + recomputing an event's own `digest`, so "serialize once ... never re-serialize on retry" holds + for the one thing that must never drift (the per-event digest, embedded as data). + + `flush()` is meant to be called repeatedly (by whatever background loop owns the call cadence, + a scheduler concern outside this module) -- each call is a complete, self-contained delivery + attempt (with its own internal retry/backoff via `_perform_with_retry`) that advances the + watermark exactly as far as the platform confirmed and leaves everything else spooled for the + next call. + + The ack body is UNTRUSTED PLATFORM INPUT end to end (v1.3): `acked_through_sequence` goes + through the spool's own M7 clamp (`advance_watermark`); `rejected[]` is filtered to sequences + actually present in the batch just sent BEFORE anything is done with it (N1) -- a value the + guest never sent cannot cause a drop, and dropping never itself advances the watermark (see + `OutboundSpool.drop_many`) -- so the batch-level `advance_watermark(acked_through_sequence)` is + the single chokepoint either way. + """ + + def __init__( + self, + capabilities: HostedCapabilities, + spool: OutboundSpool, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + batch_size: int = EVENTS_MAX_BATCH, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._spool = spool + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._batch_size = max(1, min(batch_size, EVENTS_MAX_BATCH)) + self._channel_state = channel_state or ChannelState() + + def _cap_batch(self, records: list[SpooledRecord]) -> list[SpooledRecord]: + """Proactive half of N7: cap by event count (`_batch_size`) AND cumulative canonical bytes + (`EVENTS_MAX_BATCH_BYTES`) before ever building a request -- reduces how often the reactive + 413-halving in `flush()` below is ever needed. Always includes at least one record (its own + oversized payload is HostedEventDraft's problem, at spool-append time, not this cap's).""" + capped = records[: self._batch_size] + limited: list[SpooledRecord] = [] + total_bytes = 0 + for record in capped: + if limited and total_bytes + len(record.body) > EVENTS_MAX_BATCH_BYTES: + break + limited.append(record) + total_bytes += len(record.body) + return limited + + def flush(self, *, deadline: float | None = None) -> EventsFlushResult: + self._channel_state.check() + batch = self._cap_batch(self._spool.pending_since_watermark()) + if not batch: + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ) + + response: TransportResponse | None = None + error: ChannelError | None = None + while True: + body_bytes = _encode_events_batch(batch) + headers = {**self._capabilities.auth_headers(), "Content-Type": "application/json"} + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", self._capabilities.endpoints.events, headers=headers, data=body_bytes + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + # N7: a 413 on the events channel -- reactively halve the batch and try again rather + # than returning a permanent, non-progressing error for the whole thing. Stops once a + # single event alone still 413s (defensive; should not happen under EVENT_PAYLOAD_MAX_BYTES). + if error is not None and error.status_code == 413 and len(batch) > 1: + logger.warning( + "events flush: batch of %d events (%d bytes) was rejected with 413 -- halving " + "and retrying", len(batch), len(body_bytes), + ) + batch = batch[: len(batch) // 2] + continue + break + + if error is not None or response is None: + return EventsFlushResult( + delivered_count=0, acked_through_sequence=None, rejected=[], error=error + ) + + # N2/N3: the ack body is untrusted platform input, defensively parsed -- a wrong type or a + # missing key must never raise out of flush() (that would silence the flusher loop, B1's + # outcome by a different route) and must never be treated as "0" (that would silently + # re-send the same batch forever, N3). + body = response.body if isinstance(response.body, dict) else {} + raw_acked = body.get("acked_through_sequence") + acked_through: int | None + if isinstance(raw_acked, int) and not isinstance(raw_acked, bool): + acked_through = raw_acked + else: + acked_through = None + logger.warning( + "events flush: 2xx response has a missing/invalid acked_through_sequence (got %r) " + "-- treating as a protocol violation, not advancing the watermark", raw_acked, + ) + + raw_rejected = body.get("rejected") + if isinstance(raw_rejected, list): + rejected_entries = [entry for entry in raw_rejected if isinstance(entry, dict)] + if len(rejected_entries) != len(raw_rejected): + logger.warning("events flush: rejected[] contained non-object entries -- ignoring them") + else: + rejected_entries = [] + if raw_rejected is not None: + logger.warning( + "events flush: rejected is %r, not a list -- treating as empty", type(raw_rejected).__name__ + ) + + if acked_through is None: + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ack_missing=True, + ) + + # N1: filter rejected[] to sequences the guest ACTUALLY sent in this batch, before doing + # anything with them -- a sequence the platform names that was never in `batch` is + # untrusted input this module owes no obedience to (it cannot be dropped, since it was + # never spooled under that number in the first place, and trusting it would let a + # malformed ack orphan pending records by a route the M7 clamp doesn't guard). + batch_sequences = {record.sequence for record in batch if record.sequence is not None} + valid_rejected: list[dict[str, Any]] = [] + for entry in rejected_entries: + sequence = entry.get("sequence") + if isinstance(sequence, int) and not isinstance(sequence, bool) and sequence in batch_sequences: + valid_rejected.append(entry) + else: + logger.warning( + "events flush: rejected entry names sequence=%r, which was not sent in this " + "batch (sent=%s) -- ignoring as untrusted platform input", + sequence, sorted(batch_sequences), + ) + + # P9: capture the dropped records' own bodies BEFORE drop_many physically removes them -- + # once removed, this is the only place a caller can still recover the payload the contract + # requires be "written to the artifact spool (`log` kind)" for a rejected event. + rejected_sequences = {entry["sequence"] for entry in valid_rejected} + dropped_records = [record for record in batch if record.sequence in rejected_sequences] + + # N1/N14: pure physical removal, batched into one rewrite -- drop_many never touches the + # watermark; the batch-level advance_watermark(acked_through) below is the ONLY chokepoint. + self._spool.drop_many(rejected_sequences) + + try: + # "the watermark is highest-processed -- accepted AND rejected sequences both advance + # it." v1.3: `acked_through_sequence` is untrusted input -- the spool itself enforces + # the clamp (M7). + self._spool.advance_watermark(acked_through) + except OutboundSpoolError: + logger.warning( + "events flush: platform returned an untrusted acked_through_sequence=%s outside " + "the guest's trusted range -- ignoring the ack, watermark unchanged at %s", + acked_through, self._spool.watermark(), + ) + return EventsFlushResult( + delivered_count=0, + acked_through_sequence=self._spool.watermark(), + rejected=[], + error=None, + ack_out_of_range=True, + dropped_records=dropped_records, + ) + + delivered_count = sum( + 1 + for record in batch + if record.sequence is not None + and record.sequence <= acked_through + and record.sequence not in rejected_sequences + ) + return EventsFlushResult( + delivered_count=delivered_count, + acked_through_sequence=acked_through, + rejected=valid_rejected, + error=None, + dropped_records=dropped_records, + ) + + +# ================================================================================================= +# Channel 2 -- Result receipts. Typed `ResultReceiptDraft` + delivery. +# ================================================================================================= + + +class ScenarioStatus(str, Enum): + PASSED = "passed" + FAILED = "failed" + ERRORED = "errored" + SKIPPED = "skipped" + + +class SubGoalResult(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + held: bool | None + reason: str | None + judged: bool + + +class MetricEvaluation(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal["metric"] + score: float = Field(ge=0.0, le=1.0) + reason: str + + +class CheckpointEvaluation(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1) + kind: Literal["checkpoint"] + passed: bool + reason: str + + +EvaluationResult = MetricEvaluation | CheckpointEvaluation + + +class CallSummary(BaseModel): + model_config = ConfigDict(extra="forbid") + + started_at: str + ended_at: str + duration_ms: int = Field(ge=0) + turns: int = Field(ge=0) + transcript_artifact: str | None + recording_artifacts: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def _validate(self) -> "CallSummary": + for label, value in (("started_at", self.started_at), ("ended_at", self.ended_at)): + if not is_valid_rfc3339_millis(value): + raise ValueError(f"call_timestamp_invalid: {label}={value!r}") + if self.transcript_artifact is not None and not is_valid_digest(self.transcript_artifact): + raise ValueError(f"call_transcript_artifact_invalid: {self.transcript_artifact!r}") + for artifact in self.recording_artifacts: + if not is_valid_digest(artifact): + raise ValueError(f"call_recording_artifact_invalid: {artifact!r}") + return self + + +def _unset_default_fields(model: BaseModel, prefix: str = "") -> list[str]: + """N23: names the fields a caller did NOT explicitly set (filled from a pydantic default) -- + the actual shape of a digest-mismatch bug like the review's example: a raw `call` dict that + omits `recording_artifacts` computes an external digest over an object without that key, while + the model's own re-derivation fills in `recording_artifacts: []`. This is not a general diff + against the caller's original raw dict (a model validator has no access to that, only to what + pydantic recorded via `model_fields_set`) -- it is the honest, dotted-path subset available from + inside the model: which fields with defaults were left unset, one level into nested models + (covers `call.recording_artifacts`, not just top-level `world_index`/`schema_version`).""" + names: list[str] = [] + for name in type(model).model_fields: + if name == "digest": + continue + path = f"{prefix}{name}" + if name not in model.model_fields_set: + names.append(path) + value = getattr(model, name) + if isinstance(value, BaseModel): + names.extend(_unset_default_fields(value, f"{path}.")) + return names + + +class ResultReceiptDraft(BaseModel): + """Channel 2's wire shape. Mirrors `HostedEventDraft`'s pattern: the caller supplies `digest` + (via `build_result_receipt`, computed with `whole_object_digest`), the model re-derives and + rejects a mismatch, plus the two exact-shape rules the contract states as literal requirements + rather than general validation ("`skipped` receipt body (exact)" and "`errored` receipt body"). + """ + + model_config = ConfigDict(extra="forbid") + + schema_version: str = RESULT_SCHEMA_VERSION + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + scenario_key: str = Field(min_length=1) + scenario_id: str = Field(min_length=1) + scenario_attempt: Literal[1, 2] + world_index: int | None = Field(default=None, ge=0) + status: ScenarioStatus + sub_goals: list[SubGoalResult] + evaluations: list[EvaluationResult] + call: CallSummary | None + failure: TerminalFailure | None + digest: str + + @model_validator(mode="after") + def _validate(self) -> "ResultReceiptDraft": + if self.schema_version != RESULT_SCHEMA_VERSION: + raise ValueError(f"result_schema_unsupported: {self.schema_version}") + if not is_valid_digest(self.digest): + raise ValueError(f"receipt_digest_invalid: {self.digest!r}") + expected = whole_object_digest(self.model_dump(mode="json", exclude={"digest"})) + if self.digest != expected: + unset = _unset_default_fields(self) + hint = f" -- fields not explicitly set, filled from defaults: {', '.join(unset)}" if unset else "" + raise ValueError(f"receipt_digest_mismatch{hint}") + + if self.status is ScenarioStatus.SKIPPED: + if ( + self.scenario_attempt != 1 + or self.world_index is not None + or self.sub_goals + or self.evaluations + or self.call is not None + or self.failure is not None + ): + raise ValueError("skipped_receipt_shape_invalid") + elif self.status is ScenarioStatus.ERRORED and self.failure is None: + raise ValueError("errored_receipt_requires_failure") + return self + + +def build_result_receipt( + *, + job_id: str, + attempt_id: str, + attempt_number: int, + scenario_key: str, + scenario_id: str, + scenario_attempt: Literal[1, 2], + world_index: int | None, + status: ScenarioStatus, + sub_goals: list[dict[str, Any]], + evaluations: list[dict[str, Any]], + call: dict[str, Any] | None, + failure: dict[str, Any] | None, + extra_secret_values: tuple[str, ...] = (), +) -> dict[str, Any]: + """Validate one receipt's shape and compute its digest, returning a plain wire-ready dict. + Mirrors `build_event_record`'s contract exactly: callers must pass already wire-typed values + (e.g. a metric `score` as `float`, never `int`) -- the digest is computed on the RAW input + before model validation/coercion, so a type looseness here fails loudly as a digest mismatch + rather than silently spooling a digest that doesn't match what gets sent. + + N9: `redact_outbound_text` runs on `sub_goals[].reason`, `evaluations[].reason`, and + `failure.{code,message}` (P8) BEFORE the digest is computed -- same ordering rationale as + `build_event_record`: the embedded digest must match the redacted bytes actually sent. + """ + + def _redact(text: Any) -> Any: + return redact_outbound_text(text, extra_secret_values) if isinstance(text, str) else text + + sub_goals = [ + {**goal, "reason": _redact(goal.get("reason"))} if isinstance(goal, dict) else goal + for goal in sub_goals + ] + evaluations = [ + {**item, "reason": _redact(item.get("reason"))} if isinstance(item, dict) else item + for item in evaluations + ] + if isinstance(failure, dict): + updates = { + key: _redact(failure[key]) + for key in ("code", "message") + if isinstance(failure.get(key), str) + } + if updates: + failure = {**failure, **updates} + + core: dict[str, Any] = { + "schema_version": RESULT_SCHEMA_VERSION, + "job_id": job_id, + "attempt_id": attempt_id, + "attempt_number": attempt_number, + "scenario_key": scenario_key, + "scenario_id": scenario_id, + "scenario_attempt": scenario_attempt, + "world_index": world_index, + "status": status.value if isinstance(status, ScenarioStatus) else status, + "sub_goals": sub_goals, + "evaluations": evaluations, + "call": call, + "failure": failure, + } + digest = whole_object_digest(core) + draft = ResultReceiptDraft.model_validate({**core, "digest": digest}) + return draft.model_dump(mode="json") + + +def build_skipped_receipt( + *, job_id: str, attempt_id: str, attempt_number: int, scenario_key: str, scenario_id: str +) -> dict[str, Any]: + """The "exact" synthesized body for a scenario that never ran ("The guest synthesizes these + during the flush window; the finalizer backfills any still missing").""" + return build_result_receipt( + job_id=job_id, + attempt_id=attempt_id, + attempt_number=attempt_number, + scenario_key=scenario_key, + scenario_id=scenario_id, + scenario_attempt=1, + world_index=None, + status=ScenarioStatus.SKIPPED, + sub_goals=[], + evaluations=[], + call=None, + failure=None, + ) + + +@dataclass(frozen=True) +class ReceiptPushResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +class ResultsClient: + """Delivers one Channel 2 receipt per call to `endpoints.results`. No spool/watermark of its + own -- unlike events, receipts carry no `sequence`; their idempotency key is `(job_id, + scenario_key)` (contract), so redelivery safety comes from the wire protocol itself (`200` + duplicate on a matching digest) rather than from a local ack cursor. A caller that wants + durable at-least-once delivery across a process crash owns that queuing (e.g. an + `OutboundSpool(sequenced=False)`, exposed by this module for exactly this) and simply calls + `push()` again for anything not yet confirmed -- safe because the platform's own idempotency + check is what makes a redelivery a no-op, not any state this client keeps. + """ + + def __init__( + self, + capabilities: HostedCapabilities, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._channel_state = channel_state or ChannelState() + + def push(self, receipt: dict[str, Any], *, deadline: float | None = None) -> ReceiptPushResult: + self._channel_state.check() + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", + self._capabilities.endpoints.results, + headers=self._capabilities.auth_headers(), + json_body=receipt, + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None and error.code == "attempt_superseded": # N22 + self._channel_state.latch(HostedAttemptSupersededError(error)) + if error is not None or response is None: + return ReceiptPushResult(delivered=False, already_existed=False, error=error) + # N20: `200` is read as "already exists / duplicate" per the contract's idempotency rule; + # the contract never states the success code for a genuinely NEW receipt (this module's own + # `FakePlatform` uses `201`, unconfirmed against the real platform -- see the review report). + return ReceiptPushResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + +# ================================================================================================= +# Channel 3 -- Artifacts. Content-addressed upload + `ArtifactManifestDraft` + delivery. +# ================================================================================================= + + +class ArtifactKind(str, Enum): + RECORDING_COMBINED = "recording_combined" + RECORDING_STEREO = "recording_stereo" + RECORDING_CUSTOMER = "recording_customer" + RECORDING_ASSISTANT = "recording_assistant" + TRANSCRIPT = "transcript" + TOOL_TRACE = "tool_trace" + RESULT = "result" + BUILD = "build" + TRACE = "trace" + LOG = "log" + OTHER = "other" + + +_RESERVED_ARTIFACT_KINDS = frozenset( + {ArtifactKind.BUILD, ArtifactKind.TRANSCRIPT, ArtifactKind.TOOL_TRACE, ArtifactKind.RESULT} +) + + +def is_reserved_artifact_kind(kind: ArtifactKind) -> bool: + """"the budget is partitioned by reservation: `build` + `transcript` + `tool_trace` + `result` + are reserved (always admitted); recordings next; `trace`/`log`/`other` last.\"""" + return kind in _RESERVED_ARTIFACT_KINDS + + +_RECORDING_ARTIFACT_KINDS = frozenset( + { + ArtifactKind.RECORDING_COMBINED, + ArtifactKind.RECORDING_STEREO, + ArtifactKind.RECORDING_CUSTOMER, + ArtifactKind.RECORDING_ASSISTANT, + } +) + + +def priority_class(kind: ArtifactKind) -> int: + """N16: the contract's three-tier budget partition as a total order, lower = admitted first -- + `0` reserved (`is_reserved_artifact_kind`, always admitted), `1` recordings, `2` `trace`/`log`/ + `other` (admitted last).""" + if is_reserved_artifact_kind(kind): + return 0 + if kind in _RECORDING_ARTIFACT_KINDS: + return 1 + return 2 + + +class ArtifactBudgetTracker: + """Client-side mirror of "budget = upload admission ... the guest enforces it first": a + per-job cumulative cap across attempts, deduplicated by digest. `would_admit` is a pure check a + caller makes before calling `ArtifactsClient.upload` for a non-reserved kind; when it returns + `False` the upload is skipped (and named in a `log` event -- the caller's job, not this + tracker's). This class does not sequence "refused newest-first" itself -- it has no visibility + into candidate ordering across scenarios, which only the scheduler (P9) has; it supplies the + admission arithmetic that policy is built on. + + `recording_headroom_bytes` (N16, default 0 -- no behavior change unless a caller opts in): + bytes of the remaining budget reserved for recordings not yet seen, subtracted from what a + `trace`/`log`/`other` (priority class 2) candidate is allowed to consume. This tracker has no + visibility into how many recording bytes are still coming (only the scheduler does), so it + cannot give a perfect answer -- reserving a caller-supplied headroom is the honest, testable + subset of "recordings next; trace/log/other last" this class alone can enforce. + """ + + def __init__(self, max_artifact_bytes: int, *, recording_headroom_bytes: int = 0) -> None: + self._max_bytes = max_artifact_bytes + self._admitted_bytes = 0 + self._seen_digests: set[str] = set() + self._recording_headroom_bytes = recording_headroom_bytes + + def would_admit(self, kind: ArtifactKind, size: int, *, digest: str) -> bool: + if digest in self._seen_digests: + return True # already counted; a duplicate upload never grows the budget further + tier = priority_class(kind) + if tier == 0: + return True + remaining = self._max_bytes - self._admitted_bytes + if tier == 2: + remaining -= self._recording_headroom_bytes + return size <= remaining + + def record(self, kind: ArtifactKind, size: int, *, digest: str) -> None: + del kind # reservation already resolved by would_admit; recorded uniformly here + if digest in self._seen_digests: + return + self._seen_digests.add(digest) + self._admitted_bytes += size + + @property + def admitted_bytes(self) -> int: + return self._admitted_bytes + + +class ArtifactManifestEntry(BaseModel): + model_config = ConfigDict(extra="forbid") + + artifact_id: str + kind: ArtifactKind + size: int = Field(ge=0) + scenario_key: str | None = None + + @model_validator(mode="after") + def _validate(self) -> "ArtifactManifestEntry": + if not is_valid_digest(self.artifact_id): + raise ValueError(f"artifact_id_invalid: {self.artifact_id!r}") + return self + + +class ArtifactManifestDraft(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str = MANIFEST_SCHEMA_VERSION + job_id: str = Field(min_length=1) + attempt_id: str = Field(min_length=1) + attempt_number: int = Field(ge=1) + entries: list[ArtifactManifestEntry] + complete: bool + digest: str + + @model_validator(mode="after") + def _validate(self) -> "ArtifactManifestDraft": + if self.schema_version != MANIFEST_SCHEMA_VERSION: + raise ValueError(f"manifest_schema_unsupported: {self.schema_version}") + if not is_valid_digest(self.digest): + raise ValueError(f"manifest_digest_invalid: {self.digest!r}") + expected = whole_object_digest(self.model_dump(mode="json", exclude={"digest"})) + if self.digest != expected: + unset = _unset_default_fields(self) + hint = f" -- fields not explicitly set, filled from defaults: {', '.join(unset)}" if unset else "" + raise ValueError(f"manifest_digest_mismatch{hint}") + return self + + +def build_artifact_manifest( + *, + job_id: str, + attempt_id: str, + attempt_number: int, + entries: list[dict[str, Any]], + complete: bool, +) -> dict[str, Any]: + """Same pattern as `build_result_receipt`/`build_event_record`: digest computed on the raw + input, then re-verified by the model that consumes it.""" + core: dict[str, Any] = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "job_id": job_id, + "attempt_id": attempt_id, + "attempt_number": attempt_number, + "entries": entries, + "complete": complete, + } + digest = whole_object_digest(core) + draft = ArtifactManifestDraft.model_validate({**core, "digest": digest}) + return draft.model_dump(mode="json") + + +@dataclass(frozen=True) +class ArtifactUploadResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +@dataclass(frozen=True) +class ManifestPushResult: + delivered: bool + already_existed: bool + error: ChannelError | None + + +_DEFAULT_ARTIFACT_CONTENT_TYPES: dict[ArtifactKind, str] = { + ArtifactKind.RECORDING_COMBINED: "video/mp4", + ArtifactKind.RECORDING_STEREO: "video/mp4", + ArtifactKind.RECORDING_CUSTOMER: "video/mp4", + ArtifactKind.RECORDING_ASSISTANT: "video/mp4", + ArtifactKind.TRANSCRIPT: "application/json", + ArtifactKind.RESULT: "application/json", +} + + +def _default_content_type(kind: ArtifactKind) -> str: + """N17: §3a pins recordings to mp4 and `transcript` to a JSON array; a platform serving these + back to a UI needs an accurate `Content-Type`, not a blanket octet-stream.""" + return _DEFAULT_ARTIFACT_CONTENT_TYPES.get(kind, "application/octet-stream") + + +class ArtifactsClient: + """Content-addressed upload (§3a) + manifest delivery (§3b) to `endpoints.artifacts`. + + `upload` verifies the given bytes actually hash to the claimed `artifact_id` BEFORE ever + calling the transport -- a local, zero-cost check that catches a caller bug (wrong id, wrong + bytes) without spending a round trip on it; the platform's own `422 digest_mismatch` remains + the authority for anything this local check cannot see (partial reads, transport corruption). + On a `422 digest_mismatch` FROM THE PLATFORM specifically, the contract grants exactly one + extra whole-upload retry ("re-upload once, then the referencing scenario is `errored`") -- + distinct from `_perform_with_retry`'s own loop, which treats every 422 as `PERMANENT_ITEM` and + never retries it; this method wraps that loop in one more, narrower retry layer that fires only + for that one code. + + Size accounting: `X-Artifact-Size` is never a caller-supplied value -- it is always derived + from `len(data)`, the same bytes actually transmitted, so a `422 size_mismatch` against what + this client sends is structurally unreachable from here (the platform's own count remains the + authority for what actually arrived over the wire). + + N18: once a 413 `artifact_budget_exceeded` is observed, this instance latches locally -- every + later `upload()` for a NON-reserved kind is refused without contacting the platform at all + ("stop uploading non-reserved kinds, log, continue the run"); reserved kinds keep uploading + (they are always admitted, budget or not). + """ + + def __init__( + self, + capabilities: HostedCapabilities, + transport: Transport | None = None, + *, + retry_policy: RetryPolicy | None = None, + sleep: Callable[[float], None] = time.sleep, + rng: Callable[[], float] = random.random, + chunk_threshold_bytes: int = ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES, + chunk_size_bytes: int = 8 * 1024 * 1024, + channel_state: ChannelState | None = None, + ) -> None: + self._capabilities = capabilities + self._transport = transport or RequestsTransport() + self._retry_policy = retry_policy or RetryPolicy() + self._sleep = sleep + self._rng = rng + self._chunk_threshold_bytes = chunk_threshold_bytes + self._chunk_size_bytes = chunk_size_bytes + self._channel_state = channel_state or ChannelState() + self._budget_exhausted = False # N18 + + def upload( + self, + artifact_id_hex: str, + data: bytes, + *, + kind: ArtifactKind, + scenario_key: str | None = None, + content_type: str | None = None, + deadline: float | None = None, + ) -> ArtifactUploadResult: + self._channel_state.check() + if self._budget_exhausted and not is_reserved_artifact_kind(kind): + logger.warning( + "artifacts upload: budget already exhausted (413 artifact_budget_exceeded observed " + "earlier this attempt) -- skipping non-reserved kind=%s without contacting the " + "platform", kind.value, + ) + return ArtifactUploadResult( + delivered=False, + already_existed=False, + error=ChannelError( + ChannelOutcome.BUDGET_EXCEEDED, None, "artifact_budget_exceeded", + "budget already exhausted for this attempt (latched locally)", status_code=None, + ), + ) + if not re.fullmatch(r"[0-9a-f]{64}", artifact_id_hex): + raise ValueError(f"artifact_id_invalid: {artifact_id_hex!r}") + if artifact_id_hex == "manifest": + # Unreachable through the hex check above (`manifest` isn't 64 hex chars) but the + # contract calls this collision out by name ("`manifest` is a reserved segment"). + raise ValueError("artifact_id_reserved: manifest") + computed = hashlib.sha256(data).hexdigest() + if computed != artifact_id_hex: + raise ValueError( + f"artifact_digest_mismatch_local: expected {artifact_id_hex}, computed {computed}" + ) + + url = f"{self._capabilities.endpoints.artifacts}{artifact_id_hex}/" + size = len(data) + headers = { + **self._capabilities.auth_headers(), + "X-Artifact-Kind": kind.value, + "X-Artifact-Size": str(size), + "Content-Type": content_type or _default_content_type(kind), + } + if scenario_key is not None: + headers["X-Scenario-Key"] = scenario_key + + def perform(_attempt: int) -> TransportResponse: + # A fresh generator per attempt when chunked -- see `_iter_chunks`. + body: bytes | Iterator[bytes] = ( + _iter_chunks(data, self._chunk_size_bytes) + if size > self._chunk_threshold_bytes + else data + ) + return self._transport.request("PUT", url, headers=headers, data=body) + + response: TransportResponse | None = None + error: ChannelError | None = None + for outer_attempt in range(2): # "re-upload once" on a platform-confirmed digest mismatch + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if not (error is not None and error.code == "digest_mismatch" and outer_attempt == 0): + break + + if error is not None and error.outcome is ChannelOutcome.BUDGET_EXCEEDED: + self._budget_exhausted = True # N18 + + if error is not None or response is None: + return ArtifactUploadResult(delivered=False, already_existed=False, error=error) + # N20: `200` is read as "already exists" per the contract's content-addressed upload + # semantics; the success code for a genuinely NEW upload is `201` (this module's own + # `FakePlatform` matches that but it is unconfirmed against the real platform). + return ArtifactUploadResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + def push_manifest(self, manifest: dict[str, Any], *, deadline: float | None = None) -> ManifestPushResult: + self._channel_state.check() + url = f"{self._capabilities.endpoints.artifacts}manifest/" + + def perform(_attempt: int) -> TransportResponse: + return self._transport.request( + "POST", url, headers=self._capabilities.auth_headers(), json_body=manifest + ) + + try: + response, error = _perform_with_retry( + perform, retry_policy=self._retry_policy, sleep=self._sleep, rng=self._rng, deadline=deadline + ) + except (HostedFencedError, HostedChannelFailedError) as exc: + self._channel_state.latch(exc) + raise + if error is not None and error.code == "attempt_superseded": # N22 + self._channel_state.latch(HostedAttemptSupersededError(error)) + if error is not None or response is None: + return ManifestPushResult(delivered=False, already_existed=False, error=error) + # N20: same caveat as receipts/uploads above -- `200` == duplicate is contract-stated, + # the new-manifest success code is `201` per `FakePlatform`, unconfirmed against the real + # platform. + return ManifestPushResult( + delivered=True, already_existed=response.status_code == 200, error=None + ) + + +__all__ = [ + "ARTIFACT_CHUNKED_UPLOAD_THRESHOLD_BYTES", + "CAPABILITIES_PATH", + "CAPABILITIES_SCHEMA_VERSION", + "EVENTS_MAX_BATCH", + "EVENTS_MAX_BATCH_BYTES", + "EVENT_PAYLOAD_MAX_BYTES", + "EVENT_SCHEMA_VERSION", + "FLUSH_WINDOW_SECONDS", + "MANIFEST_SCHEMA_VERSION", + "RESULT_SCHEMA_VERSION", + "ArtifactBudgetTracker", + "ArtifactKind", + "ArtifactManifestDraft", + "ArtifactManifestEntry", + "ArtifactUploadResult", + "ArtifactsClient", + "BaselineFrozenPayload", + "BaselineInputsChangedPayload", + "CallSummary", + "CapabilitiesError", + "ChannelError", + "ChannelOutcome", + "ChannelState", + "CheckpointEvaluation", + "DegradeReason", + "EvaluationResult", + "EventsClient", + "EventsFlushResult", + "HostedAttemptSupersededError", + "HostedCapabilities", + "HostedChannelFailedError", + "HostedEndpoints", + "HostedEvent", + "HostedEventDraft", + "HostedFencedError", + "LogLevel", + "LogPayload", + "ManifestPushResult", + "MetricEvaluation", + "OutboundError", + "OutboundEventType", + "OutboundSpool", + "OutboundSpoolError", + "ParallelismDegradedPayload", + "ReceiptPushResult", + "RequestsTransport", + "ResultReceiptDraft", + "ResultsClient", + "RetryPolicy", + "ScenarioCounts", + "ScenarioRetriedPayload", + "ScenarioStartedPayload", + "ScenarioStatus", + "SpooledRecord", + "StageChangedPayload", + "SubGoalResult", + "TerminalFailure", + "TerminalPayload", + "TerminalReason", + "Transport", + "TransportError", + "TransportResponse", + "WorldUnhealthyPayload", + "build_artifact_manifest", + "build_event_record", + "build_result_receipt", + "build_skipped_receipt", + "canonical_bytes", + "classify_response", + "compute_backoff_seconds", + "event_payload_digest", + "format_rfc3339_millis", + "is_reserved_artifact_kind", + "is_valid_digest", + "is_valid_rfc3339_millis", + "load_capabilities", + "priority_class", + "redact_outbound_text", + "sha256_digest", + "truncate_log_message", + "whole_object_digest", +] diff --git a/src/fi/alk/harness/process_preflight.py b/src/fi/alk/harness/process_preflight.py new file mode 100644 index 00000000..d104f0ad --- /dev/null +++ b/src/fi/alk/harness/process_preflight.py @@ -0,0 +1,673 @@ +"""The §2e pre-provision checklist — `hosted-execution-seams.md` v1.9 — as a single gate the +in-sandbox provisioner runs before starting anything. + +`bundle_v2.py` validates everything decidable from the manifest's own field values alone; this +module covers what its docstring names as deferred: the bundle's actual files on disk (digest and +per-file hashes, symlinks, path escapes, secret content), the pydantic `extra="forbid"` -> +`unknown_field` translation, and every rule that needs the job the bundle will run under +(placeholder vocabulary, secret purposes against the job's `secret_refs`, the `depends_on` graph, +the engine catalog, `seed_missing`, `inputs_digest` verification, reserved-name content scanning, +store/evidence-seam coherence, and resource sanity). `seed_strategy_unsupported`, `sentinel_shape_mismatch`, +`capability_unresolved`, `configuration_name_duplicate`, `user_assignment_invalid`, and +`capability_engine_mismatch` are already enforced by the model layer and are not repeated here. + +A missing interpreter (§0, v1.7) is a BUILD-time failure, not a preflight one — no manifest field +carries an interpreter demand, so this module has nothing to check and does not attempt to. + +`preflight_bundle` runs the checklist in the contract's own order and raises on the first +violation, never a crash — every failure is a `PreflightError` carrying a code from §2e's +failure-code table (v1.7). The caller (the provisioner) is responsible for mapping that into a +FAILED terminal state with `FailureDomain.ENVIRONMENT` in `HarnessStage.VALIDATING_ENVIRONMENT`, +per §2e's closing rule. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from pathlib import Path, PurePosixPath + +from pydantic import ValidationError + +from .artifacts import _SECRET_CONTENT, _SECRET_FILES +from .bundle import CapabilityProtocol +from .bundle_v2 import ( + BUNDLE_V2_MANIFEST, + BUNDLE_V2_SCHEMA_VERSION, + BundleFileV2, + EnvironmentBundleV2, + EvidenceSeam, + ManagedEngine, + ManagedProcess, + RuntimeKindV2, + SecretPurpose, + SourceProcess, + compute_inputs_digest, + seal_bundle_v2, +) + +_SECRET_PURPOSE_VALUES = {member.value for member in SecretPurpose} + + +class PreflightError(RuntimeError): + """A §2e checklist rule rejected the bundle. + + ``code`` is one of §2e's failure-code table (v1.9): "contract-rule" codes, each named by a + numbered checklist item's prose, and "mechanical" codes for plumbing failures the contract + describes but does not formalize as a rule (a missing bundle file, an out-of-range + ``parallelism``). Every code this module raises is in that table — including + ``fixed_port_reserved`` (F11, p5-round1-review; added to the table by v1.9), which guards + against a `fixed_port` aliasing the provisioner's own port-formula bands. + """ + + def __init__(self, code: str, message: str) -> None: + self.code = code + self.message = message + super().__init__(f"{code}: {message}") + + +_SECRET_SUFFIXES = {".pem", ".key", ".p12", ".pfx"} + +# §2b's catalog table. `ManagedEngine` already closes which *engines* exist; this closes which +# *version* of each is the one the snapshot actually ships. +_ENGINE_CATALOG_VERSION: dict[ManagedEngine, str] = { + ManagedEngine.POSTGRES: "16", + ManagedEngine.REDIS: "7", + ManagedEngine.RABBITMQ: "3.13", +} + +# §0 (v1.7): a repo needing an interpreter the snapshot lacks fails at BUILD time, reported +# `runtime_unsupported` there — not here. The manifest carries no interpreter-demand field (the +# source tree isn't embedded in the bundle, so preflight can't see `.python-version`/`engines` +# even if it wanted to), so this module has no interpreter check to run. + +# §2c: migrations/seeds must not create these — checked as a source-content scan, not a manifest +# field, since the identifier lives inside SQL/scripts the model layer never parses. +# `re.IGNORECASE`: postgres folds an unquoted identifier to lower case, so `CREATE TABLE +# _ALK_CONFORMANCE` creates the reserved table under its lower-case name — case-insensitive +# matching is the only way to catch that (F9, p4-round1-review). This is slightly over-broad for +# redis/rabbitmq, whose names are case-sensitive, but over-broad on a reserved-name check is the +# safe direction. Known false-positive surface, left as-is (documented rather than fixed): the scan +# reads whole file bytes with no lexical awareness beyond stripped `--`/`/* */` comments below, so +# a quoted string literal containing the reserved name (e.g. as inserted *data*) still trips it. +# The stripping below is a false-NEGATIVE surface in the opposite direction, equally lexer-free and +# equally left as-is: a `--` or `/*` inside a string literal (not a comment) deletes real content +# up to the next line-end or `*/`, which can delete a reserved-name definition that follows it on +# the same statement (N7, p4-round2-review). +_RESERVED_NAME = "_alk_conformance" +_RESERVED_NAME_PATTERN = re.compile( + r"(? None: + """Run the complete §2e checklist against a sealed v2 bundle directory, in the contract's own + numbered order. Raises ``PreflightError`` on the first violation; returns ``None`` when clean. + + ``manifest`` is the already-parsed model the caller obtained from ``load_bundle_v2`` — item 4 + (the pydantic ``extra_forbidden`` -> ``unknown_field`` translation bundle_v2's own docstring + defers here) is implemented by re-validating the bytes on disk, which is also where this + function's own read of ``manifest.json`` for step 1 comes from; a caller that already trusts + ``manifest`` still gets a genuine check that the file backing it hasn't drifted since. + + ``secret_refs`` maps each job secret alias to its ``SecretRef.purpose`` value (§1) — item 5's + ``secret_unclaimed``/``secret_missing`` pair needs it and the contract's own entrypoint + signature (§2e's charter) does not carry it. Required, not optional: §4's provider port hands + the provisioner ``work_directory``, and `/work/job.json` is readable from it, so every real + caller has the job's resolved refs — there is no legitimate caller that cannot supply this. + Pass ``{}`` explicitly for a job with no secret refs at all, rather than omitting the argument: + an optional default silently both under- and over-enforced the check it exists for (F2, + p4-round1-review), which required-and-explicit closes. Every value must be a ``SecretPurpose`` + value; anything else raises ``ValueError`` immediately, before any bundle content is checked. + """ + bundle_dir = Path(bundle_dir) + for alias, purpose in secret_refs.items(): + # `isinstance` first: §1's raw `agent.secret_refs` shape is `{alias: {manager, key, + # version, purpose}}`, a dict — an unhashable value would otherwise raise TypeError against + # the `in` check below instead of the ValueError this docstring promises (N8, p4-round2- + # review). + if not isinstance(purpose, str) or purpose not in _SECRET_PURPOSE_VALUES: + raise ValueError(f"secret_refs[{alias!r}] = {purpose!r} is not a SecretPurpose value") + + if manifest.runtime.kind is RuntimeKindV2.COMPOSE: + # §2a: "a hosted job with kind: compose fails preflight" — not one of §2e's seven numbered + # items, so ahead of item 1 rather than slotted between them: every item below assumes + # v2's processes/seed shape, which a compose bundle need not carry, and a compose bundle's + # own files (its document, e.g.) carry no obligation to be exhaustively listed in files[] + # the way a hosted bundle's do — checking file-listing first mis-reported that case as + # bundle_file_unlisted instead of compose_not_hosted (N1, p4-round2-review). + raise PreflightError("compose_not_hosted", "kind: compose is not a legal hosted runtime") + + files = _verify_digest(bundle_dir, manifest) # 1 + walked_files = _verify_path_safety(bundle_dir, files) # 2 + _scan_bundle_files_for_secrets(bundle_dir, walked_files) # 3 + _verify_unknown_fields(bundle_dir, manifest) # 4 + + if manifest.runtime.kind is RuntimeKindV2.PROCESS: + _verify_placeholder_vocabulary(manifest) # 5 + _verify_no_root_build_commands(manifest) # 5 (§2a) + _verify_secret_purposes(manifest, secret_refs) # 5 + _verify_depends_on(manifest) # 5 + _verify_engine_catalog(manifest) # 5 + _verify_fixed_port_not_reserved(manifest) # 5 / §2b + _verify_seed_missing(manifest) # 5 / §2c + _verify_reserved_names(bundle_dir, manifest) # 5 + _verify_seed_files_on_disk_and_listed(bundle_dir, manifest, files) # 5 + + _verify_no_sql_store(manifest) # 6 (relaxed: storeless bundles are legal) + _verify_resource_sanity(manifest, parallelism=parallelism) # 7 + + +# --- item 1: digest verification ------------------------------------------------------------- + + +def _verify_digest(bundle_dir: Path, manifest: EnvironmentBundleV2) -> list[BundleFileV2]: + root = bundle_dir.resolve() + try: + raw = json.loads((root / BUNDLE_V2_MANIFEST).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError("bundle_manifest_invalid", str(exc)) from exc + on_disk_schema_version = raw.get("schema_version") if isinstance(raw, dict) else None + if on_disk_schema_version != BUNDLE_V2_SCHEMA_VERSION: + # §2e item 1 opens with "schema_version is …bundle.v2" — checked here, at item 1, + # rather than left to surface three items late through item 4's re-validation fallback + # (F11, p4-round1-review). + raise PreflightError("bundle_schema_unsupported", str(on_disk_schema_version)) + for record in manifest.files: + path = root / record.path + if not path.is_file(): + raise PreflightError("bundle_file_missing", record.path) + digest = hashlib.sha256() + size = 0 + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + if digest.hexdigest() != record.sha256 or size != record.size: + raise PreflightError("bundle_file_changed", record.path) + recomputed = seal_bundle_v2(manifest) + if recomputed != manifest.digest: + raise PreflightError( + "bundle_digest_mismatch", f"expected {manifest.digest}, computed {recomputed}" + ) + return manifest.files + + +# --- item 2: path safety on the filesystem itself ---------------------------------------------- + + +def _verify_path_safety(bundle_dir: Path, files: list[BundleFileV2]) -> list[Path]: + """The model already rejects unsafe strings in `files[].path` (`_safe_relative`); this walks + the actual filesystem, which a string check cannot: a symlinked directory can make an + innocent-looking relative path resolve outside the bundle root. + + Every non-directory entry except the bundle root's own `manifest.json` must be recorded in + `files[]` (`bundle_file_unlisted`) — a file physically present but never listed was invisible + to both the digest check above and the secret scan that follows, which is exactly what let an + unlisted `.env` through undetected (F1, p4-round1-review). The `manifest.json` exemption is by + exact root path, not by basename (F10, p4-round1-review): a nested `db/manifest.json` gets no + special treatment, only `bundle_dir/manifest.json` itself. The exemption covers only the + listing check, not the symlink check — a symlinked root `manifest.json` would otherwise be + waved through here and then read straight through by `_verify_digest`/`_verify_unknown_fields`, + the very item whose job is to stop path escapes (N3, p4-round2-review). + + Returns the walked file paths so the secret scan (item 3) can run against what the filesystem + actually contains rather than against `files[]` again. + """ + root = bundle_dir.resolve() + manifest_path = root / BUNDLE_V2_MANIFEST + listed = {record.path for record in files} + walked: list[Path] = [] + for entry in root.rglob("*"): + if entry.is_symlink(): + raise PreflightError("bundle_symlink_forbidden", str(entry.relative_to(root))) + if entry == manifest_path: + continue + if entry.is_dir(): + continue + relative = entry.relative_to(root).as_posix() + if relative not in listed: + raise PreflightError("bundle_file_unlisted", relative) + walked.append(entry) + return walked + + +# --- item 3: secret material in the bundle's own files ------------------------------------------ + + +def _scan_bundle_files_for_secrets(bundle_dir: Path, walked_files: list[Path]) -> None: + """Reuses `artifacts.py`'s own file-name and content secret scan unchanged — the same + high-entropy-token regexes and credential-file-name set this codebase already applies to + sealed run artifacts, applied here to a sealed bundle's files instead. + + Scoped to every file item 2's filesystem walk actually found, not to `files[]` (F1, + p4-round1-review) — an unlisted secret file is already rejected by item 2's own + `bundle_file_unlisted` check, but this scan must not depend on that running first to be + correct on its own terms. + """ + root = bundle_dir.resolve() + for path in walked_files: + relative = path.relative_to(root).as_posix() + posix_path = PurePosixPath(relative) + if posix_path.name in _SECRET_FILES or posix_path.suffix.lower() in _SECRET_SUFFIXES: + raise PreflightError("secret_in_bundle", f"{relative}: forbidden secret-shaped file") + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + if any(pattern.search(chunk) for pattern in _SECRET_CONTENT): + raise PreflightError( + "secret_in_bundle", f"{relative}: high-entropy secret-scan hit" + ) + + +# --- item 4: unknown-field translation ---------------------------------------------------------- + + +def _verify_unknown_fields(bundle_dir: Path, manifest: EnvironmentBundleV2) -> None: + target = bundle_dir / BUNDLE_V2_MANIFEST + try: + raw = json.loads(target.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise PreflightError("bundle_manifest_invalid", str(exc)) from exc + try: + revalidated = EnvironmentBundleV2.model_validate(raw) + except ValidationError as exc: + raise _translate_validation_error(exc) from exc + if revalidated.model_dump(mode="json") != manifest.model_dump(mode="json"): + # Re-validating catches drift that makes the file *invalid*; it says nothing about drift + # that leaves it valid (a changed `run_command`, a flipped `user`) unless the two dumps are + # actually compared (F12, p4-round1-review). + raise PreflightError( + "bundle_manifest_drifted", "manifest.json on disk no longer matches manifest argument" + ) + + +def _translate_validation_error(exc: ValidationError) -> PreflightError: + """§2b: "unknown keys in a process entry are a preflight error (`unknown_field`)" — the model + layer's docstring defers this exact translation here, since pydantic's own `extra_forbidden` + carries no contract vocabulary of its own. Every other model-layer rejection already embeds + its own snake_case code as the leading token of its message (see `bundle_v2.py`'s + `model_validator`s); that code is preserved rather than collapsed into a generic one. + """ + for error in exc.errors(): + if error.get("type") == "extra_forbidden": + location = ".".join(str(part) for part in error["loc"]) + return PreflightError("unknown_field", f"{location}: unknown field") + # `(?::|$)`, not just `:` (F13, p4-round1-review): a bare code with no trailing detail (e.g. + # `bundle_digest_invalid`) is the entire message, with nothing after it to require a colon + # before. Scans every error, not just the first, since pydantic's own ordering is not the + # contract's priority — the first message that yields a recognizable code wins. + for error in exc.errors(): + message = str(error.get("msg", "")) + matched = re.match(r"(?:Value error, )?([a-z][a-z0-9_]*)(?::|$)", message) + if matched: + return PreflightError(matched.group(1), message) + return PreflightError("bundle_manifest_invalid", str(exc)) + + +# --- item 5: everything the model layer needs the job or the files for ------------------------ + + +def _verify_placeholder_vocabulary(manifest: EnvironmentBundleV2) -> None: + """§2b's closed `{{...}}` vocabulary, checked in `environment`. `build_environment` takes NO + placeholders at all (§2b) — any `{{...}}` match there is rejected outright, never resolved + against the vocabulary below (F6, p4-round1-review). + + `{{}}` can only ever resolve to a capability whose `configuration_name` + is non-null — a capability left null is therefore structurally unreachable by any placeholder, + which is what makes this scan also enforce §2d's "non-null whenever referenced by any process + `environment`... entry" without a second pass. When the unmatched token is exactly a declared + capability's slug and that capability's `configuration_name` is null, the real problem is the + missing name, not the token — reported `capability_unresolved` naming the capability, rather + than the generic `unknown_placeholder` every other unmatched token gets (F15, p4-round1-review; + a deliberate resolution — §2d names no other string a producer could have meant). + """ + known_names = {process.name for process in manifest.processes} + known_configuration_names = { + capability.configuration_name + for capability in manifest.capabilities.values() + if capability.configuration_name + } + unresolved_capability_slugs = { + slug + for slug, capability in manifest.capabilities.items() + if not capability.configuration_name + } + for process in manifest.processes: + if not isinstance(process, SourceProcess): + continue + for key, value in (process.build_environment or {}).items(): + match = _PLACEHOLDER.search(value) + if match: + raise PreflightError( + "unknown_placeholder", + f"{process.name}.build_environment.{key}: {{{{{match.group(1)}}}}} — " + "build_environment takes no placeholders", + ) + for key, value in process.environment.items(): + for match in _PLACEHOLDER.finditer(value): + token = match.group(1) + if token in _FIXED_PLACEHOLDERS: + continue + named = _NAMED_PLACEHOLDER.match(token) + if named: + _, name = named.groups() + if name in known_names: + continue + raise PreflightError( + "unknown_placeholder", + f"{process.name}.environment.{key}: {{{{{token}}}}} names an unknown " + "process", + ) + if token in known_configuration_names: + continue + if token in unresolved_capability_slugs: + raise PreflightError( + "capability_unresolved", + f"{process.name}.environment.{key}: {{{{{token}}}}} names capability " + f"{token!r}, which has no configuration_name", + ) + raise PreflightError( + "unknown_placeholder", + f"{process.name}.environment.{key}: {{{{{token}}}}} is not in the closed " + "placeholder vocabulary", + ) + + +def _verify_no_root_build_commands(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, SourceProcess): + continue + for step in process.build_commands: + if step[0] in _ROOT_BUILD_COMMANDS or "sudo" in step: + raise PreflightError( + "build_requires_root", f"{process.name}: build step {step!r} requires root" + ) + + +def _verify_secret_purposes(manifest: EnvironmentBundleV2, secret_refs: dict[str, str]) -> None: + """§2b: both directions, scoped to `target_provider` only — `source_checkout` and any other + gateway-only purpose never crosses into the guest (§0 step 3) and has nothing to claim here.""" + ref_has_target_provider = any( + purpose == SecretPurpose.TARGET_PROVIDER.value for purpose in secret_refs.values() + ) + process_claims_target_provider = any( + SecretPurpose.TARGET_PROVIDER in process.secret_purposes + for process in manifest.processes + if isinstance(process, SourceProcess) + ) + if ref_has_target_provider and not process_claims_target_provider: + raise PreflightError( + "secret_unclaimed", "a target_provider secret ref is not listed by any process" + ) + if process_claims_target_provider and not ref_has_target_provider: + raise PreflightError( + "secret_missing", "a process lists secret_purposes: target_provider but the job " + "supplies no such ref" + ) + + +def _verify_depends_on(manifest: EnvironmentBundleV2) -> None: + graph = {process.name: list(process.depends_on) for process in manifest.processes} + for name, deps in graph.items(): + unknown = sorted(dep for dep in deps if dep not in graph) + if unknown: + raise PreflightError( + "depends_on_unresolved", + f"{name} depends_on unknown process(es): {', '.join(unknown)}", + ) + + unvisited, in_progress, done = 0, 1, 2 + state = {name: unvisited for name in graph} + + def visit(name: str, stack: list[str]) -> None: + state[name] = in_progress + stack.append(name) + for dep in graph[name]: + if state[dep] == in_progress: + cycle = stack[stack.index(dep):] + [dep] + raise PreflightError("depends_on_cycle", " -> ".join(cycle)) + if state[dep] == unvisited: + visit(dep, stack) + stack.pop() + state[name] = done + + for name in sorted(graph): + if state[name] == unvisited: + visit(name, []) + + +def _verify_engine_catalog(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + pinned = _ENGINE_CATALOG_VERSION[process.engine] + if process.version != pinned: + raise PreflightError( + "engine_unsupported", + f"{process.name}: {process.engine.value} {process.version} is not supported; " + f"the snapshot ships {process.engine.value} {pinned}", + ) + + +def _verify_fixed_port_not_reserved(manifest: EnvironmentBundleV2) -> None: + for process in manifest.processes: + if not isinstance(process, SourceProcess) or process.fixed_port is None: + continue + if ( + process.fixed_port in _JOB_SHARED_PORT_BAND + or process.fixed_port in _PER_WORLD_PORT_BAND + or process.fixed_port in _JOB_SHARED_RABBITMQ_MANAGEMENT_BAND + or process.fixed_port in _PER_WORLD_RABBITMQ_MANAGEMENT_BAND + ): + raise PreflightError( + "fixed_port_reserved", + f"{process.name}: fixed_port {process.fixed_port} falls inside the provisioner's " + "own port-formula bands (14000-14099 job-shared, 15000-15799 per-world, " + "24000-24099/25000-25799 rabbitmq management)", + ) + + +def _verify_seed_missing(manifest: EnvironmentBundleV2) -> None: + covered = {store.capability for store in (manifest.seed.stores if manifest.seed else [])} + missing = sorted( + slug + for slug, capability in manifest.capabilities.items() + if capability.protocol is CapabilityProtocol.POSTGRES and slug not in covered + ) + if missing: + raise PreflightError( + "seed_missing", + "postgres-protocol capability with no store entry: " + ", ".join(missing), + ) + + +def _verify_reserved_names(bundle_dir: Path, manifest: EnvironmentBundleV2) -> None: + if manifest.seed is None: + return + root = bundle_dir.resolve() + for store in manifest.seed.stores: + for relative_path in (*store.migrations, *store.seed_files): + path = root / relative_path + if not path.is_file(): + continue # reported by `_verify_seed_files_on_disk_and_listed` + text = path.read_text(encoding="utf-8", errors="replace") + # Strip `--`-to-EOL and `/* ... */` comments before scanning (F9, p4-round1-review) — + # a generated seed file's own note about the reservation ("-- never create + # _alk_conformance here") would otherwise trip the scan on prose, not on an identifier + # it defines. Quoted string literals containing the name as *data* remain a known + # false-positive surface: the scan has no lexer, only comment-stripping. The stripping + # is a false-NEGATIVE surface in the opposite direction, equally lexer-free and equally + # left as-is (N7, p4-round2-review; B4, p4-round3-review): a `--` or `/*` inside a + # string literal (not a comment) deletes real content up to the next line-end or `*/`, + # which can delete a reserved-name definition that follows it on the same statement. + code = _SQL_BLOCK_COMMENT.sub("", _SQL_LINE_COMMENT.sub("", text)) + if _RESERVED_NAME_PATTERN.search(code): + raise PreflightError( + "reserved_name", + f"{relative_path} defines the reserved conformance-canary identifier " + f"{_RESERVED_NAME!r}", + ) + + +def _verify_seed_files_on_disk_and_listed( + bundle_dir: Path, manifest: EnvironmentBundleV2, files: list[BundleFileV2] +) -> None: + """Digest verification (item 1) already guarantees every ``files[]``-listed path exists, so a + path missing from disk entirely is ``seed_file_missing`` regardless of whether it was ever + listed. ``seed_file_unlisted`` stays here as a second, store-scoped statement of the same + "listed" rule item 2's own walk now enforces bundle-wide (F1, p4-round1-review) — through + `preflight_bundle`'s full sequence item 2's ``bundle_file_unlisted`` always fires first for any + file the walk visits; the root ``manifest.json`` is exempt from that walk, so a store path + naming it still reaches here (N2, p4-round2-review). + + Once every migration/seed file for a store is confirmed present and listed, its recorded + ``inputs_digest`` is recomputed and compared (F14, p4-round1-review; §2c makes it the baseline + identity attempt-retry reuse trusts absolutely, and nothing else on either side of the seam + ever validated it). ``engine``/``version`` come from the store's capability's own backing + ``ManagedProcess`` — guaranteed to exist by `bundle_v2`'s ``store_service_not_managed`` check; + a non-``ManagedProcess`` backing here would mean that guarantee broke, raised as a typed + ``PreflightError`` rather than asserted, since this module's charter is a rejection on every + path, never a crash (N8, p4-round2-review). + """ + if manifest.seed is None: + return + listed = {record.path for record in files} + root = bundle_dir.resolve() + processes_by_name = {process.name: process for process in manifest.processes} + for store in manifest.seed.stores: + for relative_path in (*store.migrations, *store.seed_files): + if not (root / relative_path).is_file(): + raise PreflightError("seed_file_missing", f"{relative_path} does not exist on disk") + if relative_path not in listed: + raise PreflightError( + "seed_file_unlisted", f"{relative_path} is not listed in files[]" + ) + capability = manifest.capabilities[store.capability] + engine_process = processes_by_name[capability.service] + if not isinstance(engine_process, ManagedProcess): + raise PreflightError( + "store_service_not_managed", + f"{store.capability}: service {capability.service!r} is not a managed engine", + ) + recomputed = compute_inputs_digest( + root, + store.migrations, + store.seed_files, + engine=engine_process.engine, + version=engine_process.version, + ) + if recomputed != store.baseline.inputs_digest: + raise PreflightError( + "inputs_digest_mismatch", + f"{store.capability}: expected {store.baseline.inputs_digest}, computed " + f"{recomputed}", + ) + + +# --- item 6: store / evidence-seam coherence ----------------------------------------------------- + + +def _verify_no_sql_store(manifest: EnvironmentBundleV2) -> None: + """A `kind: process` bundle with zero postgres-protocol capabilities is a legal stateless + agent: its worlds hold no state, and only scenario code that actually asks for state gets a + typed failure (`ProcessWorldFactory`'s storeless handle). The one thing a storeless bundle can + never satisfy is `evidence_seam: tool_trace` — that seam reads the agent's tool calls out of + the world's own postgres database, so accepting it here would guarantee zero evidence at + runtime and fail every scenario with `evidence_missing` instead of a preflight verdict.""" + if manifest.runtime.kind is not RuntimeKindV2.PROCESS: + return + if any( + capability.protocol is CapabilityProtocol.POSTGRES + for capability in manifest.capabilities.values() + ): + return + if manifest.runtime.evidence_seam is EvidenceSeam.TOOL_TRACE: + raise PreflightError( + "evidence_seam_unsatisfiable", + "evidence_seam: tool_trace reads evidence from a postgres store, and this bundle " + "declares no postgres-protocol capability", + ) + + +# --- item 7: resource sanity ---------------------------------------------------------------------- + + +def _verify_resource_sanity(manifest: EnvironmentBundleV2, *, parallelism: int) -> None: + if len(manifest.processes) > _MAX_PROCESSES: + raise PreflightError( + "process_count_exceeded", + f"{len(manifest.processes)} processes exceeds the {_MAX_PROCESSES} cap", + ) + if not (_MIN_PARALLELISM <= parallelism <= _MAX_PARALLELISM): + raise PreflightError( + "parallelism_out_of_range", + f"parallelism={parallelism} is outside {_MIN_PARALLELISM}..{_MAX_PARALLELISM}", + ) + if parallelism > 1: + # Concurrent worlds dial their agents by LiveKit identity. A static + # agent name registers every world's agent under the same identity, so + # a dispatch lands on an arbitrary world — evidence silently crosses + # worlds. The {{WORLD_INDEX}} placeholder is what makes names per-world. + for process in manifest.processes: + environment = getattr(process, "environment", None) or {} + value = environment.get("LIVEKIT_AGENT_NAME") + if isinstance(value, str) and value and "{{WORLD_INDEX}}" not in value: + raise PreflightError( + "agent_name_not_world_unique", + f"process {process.name} renders a static LIVEKIT_AGENT_NAME " + f"with parallelism={parallelism}; include {{{{WORLD_INDEX}}}}", + ) + + +__all__ = ["PreflightError", "preflight_bundle"] diff --git a/src/fi/alk/harness/process_runtime.py b/src/fi/alk/harness/process_runtime.py new file mode 100644 index 00000000..af26ab11 --- /dev/null +++ b/src/fi/alk/harness/process_runtime.py @@ -0,0 +1,4181 @@ +"""The execution half of the in-sandbox provisioner — `hosted-execution-seams.md` v1.12, §2b/§3/§4/§5. + +Builds one world's running processes from an already-`preflight_bundle`-cleared +`EnvironmentBundleV2`: port allocation, `{{...}}` placeholder rendering, copy-based build trees, +process spawn (managed engines and `source` processes), `depends_on` wait, and `healthy()` +readiness probing. This is `provision()` UP TO process spawn only — seed/baseline application, +the world-reset mechanism, and the conformance gate (§4) are a later phase, deliberately left out +so they compose on top of `spawn_world`/`build_process_tree` rather than reworking them. The +`RuntimeProvider` Protocol itself (`runtime.py`) is untouched here; a later phase wires a +Protocol-conforming, stateful adapter around the pure functions below. + +Nothing here assumes Docker, a container runtime, or a network provider — §0: "No Docker inside +the sandbox." Every engine/process concept is a plain `subprocess`, matched against the module's +own `ProcessRunner`/`CapabilityProber` seams so tests substitute fakes; the one exception +(`default_capability_prober`'s postgres branch) import-guards `psycopg` and falls back to a bare +TCP probe when it is absent, the same fallback a bundle's own store would get before a role or +database exists to authenticate against. + +§0/§2b `user`: every build tree and every spawned process is chowned/spawned under its declared +`user`, resolved through `pwd.getpwnam` (`default_user_resolver`). A dev box has none of the +snapshot's `svc-*` accounts, so resolution failing is the expected local-lane shape, not an error +by itself — `require_declared_user=False` (the default) falls back to running unprivileged and +logs it; a hosted caller that wants a missing user to be a typed failure instead sets it `True`. +""" + +from __future__ import annotations + +import base64 +import json +import logging +import os +import pwd +import re +import secrets +import shutil +import signal +import socket +import subprocess +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, replace +from enum import Enum +from pathlib import Path +from typing import Any, Callable, Protocol, Sequence +from urllib.parse import quote as urlquote, urlsplit + +from pydantic import BaseModel, Field, JsonValue + +from .bundle import CapabilityProtocol +from .bundle_v2 import ( + BaselineStrategy, + CapabilityV2, + EnvironmentBundleV2, + ManagedEngine, + ManagedProcess, + ProcessUser, + ReadinessProbeV2, + SecretPurpose, + SourceProcess, + StoreEntry, +) +from .job import FailureDomain + +logger = logging.getLogger(__name__) + +# --- errors -------------------------------------------------------------------------------- + + +class ProcessRuntimeError(RuntimeError): + """A runtime-execution failure below the §2e preflight gate. + + The bundle has already passed `preflight_bundle` by the time any of this module runs, so + these are process/filesystem/timing failures. v1.8 added §2f: a CLOSED table for the subset + of these that cross the outbound seam (`source_tree_unavailable`, `build_failed`, + `runtime_unsupported`, `spawn_failed`, `depends_on_timeout`, `unsupported_capability_protocol`), + each mapped to a `FailureDomain` per §4.6 — every code this module raises THAT crosses the + seam is in that table. A handful of `code` values prefixed `internal_` + (`internal_unknown_placeholder`, `internal_missing_credentials`) are deliberately NOT in it: + each marks a precondition `preflight_bundle` should already have made impossible, so it is a + bug to fix here, not a failure the outbound seam ever needs a name for. `stage` names which + phase failed (`build`, `spawn`, `depends_on`, `render`); `process` names the process involved, + when there is one. + + v1.15 §2f: the PRODUCER resolves `domain` — a row like `spawn_failed` splits managed-vs-source + on a fact (the failing process's declared `kind`) only known at the raise site, so every raise + site for a §2f code sets `domain` here rather than leaving the consumer to re-derive it from + `code` alone (which cannot recover the managed/source split). `domain` is `None` only where + the raise site genuinely cannot tell which process kind failed (a mixed-kind catch-all); such + sites are rare and `SECTION_2F_DOMAIN` below exists as their fallback. + """ + + def __init__( + self, + stage: str, + code: str, + message: str, + *, + process: str | None = None, + domain: FailureDomain | None = None, + ) -> None: + self.stage = stage + self.code = code + self.process = process + self.domain = domain + located = f" ({process})" if process else "" + super().__init__(f"{stage}/{code}{located}: {message}") + + +# §2f's closed build/run failure-code table (hosted-execution-seams.md §4.6) -> FailureDomain. +# THE single home for this map (v1.15's §2f-map cleanup) — `hosted_scheduler.py`/ +# `hosted_entrypoint.py` import it rather than keeping their own copies. It is a FALLBACK ONLY: +# every raise site above sets `ProcessRuntimeError.domain` directly, so a consumer reads that +# first and falls back to this map only for an error that reaches it with no carried domain. +# `spawn_failed`'s entry is the pre-v1.15 conservative guess (matches its source-process half); +# a `spawn_failed` raised with a resolved `domain` never consults this entry at all. +SECTION_2F_DOMAIN: dict[str, FailureDomain] = { + "source_tree_unavailable": FailureDomain.ENVIRONMENT, + "build_failed": FailureDomain.AGENT, + "runtime_unsupported": FailureDomain.ENVIRONMENT, + "spawn_failed": FailureDomain.INFRASTRUCTURE, + "depends_on_timeout": FailureDomain.INFRASTRUCTURE, + "unsupported_capability_protocol": FailureDomain.ENVIRONMENT, + "seed_failed": FailureDomain.ENVIRONMENT, + "store_statement_failed": FailureDomain.INFRASTRUCTURE, +} + + +# --- §3 EnvironmentRuntime ------------------------------------------------------------------- + + +class RuntimeState(str, Enum): + PREPARING = "preparing" + READY = "ready" + UNHEALTHY = "unhealthy" + STOPPED = "stopped" + + +class RuntimeEndpoint(BaseModel): + capability: str + protocol: str + address: str + configuration_name: str | None = None + + +class EnvironmentRuntime(BaseModel): + """§3's per-world provisioner output. No `provider` field (§4 delta: "no remote provider + exists at this seam") — the class attribute `RuntimeProvider.name` (`runtime.py`) is retained + there for logging only, not carried onto each world's own record.""" + + runtime_id: str + world_index: int = Field(ge=0) + bundle_digest: str + state: RuntimeState + endpoints: dict[str, RuntimeEndpoint] = Field(default_factory=dict) + metadata: dict[str, JsonValue] = Field(default_factory=dict) + + +def new_runtime_id(bundle_digest: str, world_index: int) -> str: + """Opaque per §3 ("nothing parses it") — still traceable in logs without decoding.""" + return f"{bundle_digest}:w{world_index}:{secrets.token_hex(4)}" + + +# --- §2b port allocation --------------------------------------------------------------------- + +_PER_WORLD_BASE = 15000 +_PER_WORLD_STRIDE = 100 +_JOB_SHARED_BASE = 14000 + + +def _ordinal_map(manifest: EnvironmentBundleV2) -> dict[str, int]: + """§2b: "ordinal = the process's 0-based index in the `processes` array as authored" — one + map, shared by both port ranges.""" + return {process.name: index for index, process in enumerate(manifest.processes)} + + +def _job_shared_process_names(manifest: EnvironmentBundleV2) -> frozenset[str]: + """§2b instancing rule: a managed engine runs once per job iff some store backed by one of + its capabilities uses `template_database`; `datadir_copy`, `empty`, or no `seed.stores` entry + at all is per-world — per-world is the only safe default for an engine one world's reset + could otherwise corrupt for the others. Only postgres's catalog entry permits + `template_database` at all (`bundle_v2._ENGINE_STRATEGIES`), so this can never mark a redis + or rabbitmq process job-shared. + """ + service_by_capability = {slug: cap.service for slug, cap in manifest.capabilities.items()} + strategies_by_service: dict[str, set[BaselineStrategy]] = {} + if manifest.seed is not None: + for store in manifest.seed.stores: + service = service_by_capability.get(store.capability) + if service is not None: + strategies_by_service.setdefault(service, set()).add(store.baseline.strategy) + return frozenset( + process.name + for process in manifest.processes + if isinstance(process, ManagedProcess) + and BaselineStrategy.TEMPLATE_DATABASE in strategies_by_service.get(process.name, set()) + ) + + +@dataclass(frozen=True) +class PortPlan: + """A job's whole port assignment, computed once from the manifest and the requested instance + count. `fixed_port` (§2b) forces `effective_instances` to 1 — `port_for` honors the fixed + value exactly for that process, in every world index, which is consistent because there can + only ever be world 0 once `effective_instances` is 1. Emitting the `parallelism_degraded` + event for `degraded_reason` is the scheduler's job (§2b), not this module's — this only + records the fact. + """ + + ordinals: dict[str, int] + job_shared: frozenset[str] + fixed_ports: dict[str, int] + effective_instances: int + degraded_reason: str | None + + def is_job_shared(self, process_name: str) -> bool: + return process_name in self.job_shared + + def port_for(self, process_name: str, world_index: int) -> int: + if process_name in self.fixed_ports: + return self.fixed_ports[process_name] + ordinal = self.ordinals[process_name] + if process_name in self.job_shared: + return _JOB_SHARED_BASE + ordinal + return _PER_WORLD_BASE + _PER_WORLD_STRIDE * world_index + ordinal + + +def plan_ports(manifest: EnvironmentBundleV2, *, instances: int) -> PortPlan: + ordinals = _ordinal_map(manifest) + job_shared = _job_shared_process_names(manifest) + fixed_ports = { + process.name: process.fixed_port + for process in manifest.processes + if isinstance(process, SourceProcess) and process.fixed_port is not None + } + return PortPlan( + ordinals=ordinals, + job_shared=job_shared, + fixed_ports=fixed_ports, + effective_instances=1 if fixed_ports else instances, + degraded_reason="fixed_port" if fixed_ports and instances > 1 else None, + ) + + +# --- engine credentials, generated once per job ----------------------------------------------- + + +@dataclass(frozen=True) +class EngineCredentials: + username: str + password: str + + +def generate_engine_credentials( + manifest: EnvironmentBundleV2, *, token: Callable[[], str] | None = None +) -> dict[str, EngineCredentials]: + """§2b catalog: postgres/rabbitmq use role/user `harness`, "password generated per job"; + redis carries no auth in V1 and gets no entry here. Keyed by process name (not engine) since + a job can carry more than one postgres process. `token` is injectable for deterministic + tests; production callers rely on the `secrets` module default. + """ + make_token = token or (lambda: secrets.token_urlsafe(24)) + return { + process.name: EngineCredentials(username="harness", password=make_token()) + for process in manifest.processes + if isinstance(process, ManagedProcess) + and process.engine in (ManagedEngine.POSTGRES, ManagedEngine.RABBITMQ) + } + + +def render_capability_address( + capability: CapabilityV2, + *, + port: int, + world_index: int, + credentials: EngineCredentials | None, +) -> str: + """§2b: `{{DATABASE_URL}}` etc. render "with the catalog role, the generated password, the + allocated port, and `{{DB_NAME}}`." `{{DB_NAME}}` is always `w` (§2b), including under + `template_database`, where the logical database differs per world on one shared server.""" + host = "localhost" + protocol = capability.protocol + if protocol is CapabilityProtocol.POSTGRES: + if credentials is None: + # S11/P4's N8(2) precedent: a bare `assert` for a real precondition is stripped under + # `python -O` — this is a should-never-happen internal invariant (`generate_engine_ + # credentials` always produces one for every postgres/rabbitmq `ManagedProcess`), not + # a bundle defect, so it is `internal_`-prefixed, not a §2f code. + raise ProcessRuntimeError( + "render", "internal_missing_credentials", + "postgres capability requires generated credentials but none were supplied", + ) + auth = f"{credentials.username}:{credentials.password}" + return f"postgresql://{auth}@{host}:{port}/w{world_index}" + if protocol is CapabilityProtocol.AMQP: + if credentials is None: + raise ProcessRuntimeError( + "render", "internal_missing_credentials", + "amqp capability requires generated credentials but none were supplied", + ) + return f"amqp://{credentials.username}:{credentials.password}@{host}:{port}/" + if protocol is CapabilityProtocol.REDIS: + return f"redis://{host}:{port}" + if protocol is CapabilityProtocol.HTTP: + return f"http://{host}:{port}" + # F10, p5-round1-review: no other capability protocol (grpc, mongodb, s3, ...) has a defined + # address shape at this seam — §3 names exactly two worked examples. A bare + # `://host:port` used to be handed to a customer process as its own + # `{{CONFIGURATION_NAME}}` value, which is not a working address for any of those protocols; + # failing loudly beats handing out an address that cannot work. + raise ProcessRuntimeError( + "render", "unsupported_capability_protocol", + f"{protocol.value} has no defined address shape at this seam", + domain=FailureDomain.ENVIRONMENT, + ) + + +def build_endpoints( + manifest: EnvironmentBundleV2, + *, + world_index: int, + port_plan: PortPlan, + credentials: dict[str, EngineCredentials], +) -> dict[str, RuntimeEndpoint]: + """§3's `endpoints` map, built fresh for one world — every declared capability gets an + entry regardless of whether any process placeholder ever references its `configuration_name` + (a null `configuration_name` is legal; the scheduler/simulator still read endpoints by + capability slug, per §3's own reader list).""" + endpoints: dict[str, RuntimeEndpoint] = {} + for slug, capability in manifest.capabilities.items(): + port = port_plan.port_for(capability.service, world_index) + address = render_capability_address( + capability, + port=port, + world_index=world_index, + credentials=credentials.get(capability.service), + ) + endpoints[slug] = RuntimeEndpoint( + capability=slug, + protocol=capability.protocol.value, + address=address, + configuration_name=capability.configuration_name, + ) + return endpoints + + +def configuration_addresses_from_endpoints( + endpoints: dict[str, RuntimeEndpoint], +) -> dict[str, str]: + return { + endpoint.configuration_name: endpoint.address + for endpoint in endpoints.values() + if endpoint.configuration_name + } + + +# --- §2b placeholder renderer ------------------------------------------------------------------ + +_PLACEHOLDER = re.compile(r"\{\{([^{}]+)\}\}") +_NAMED_PLACEHOLDER = re.compile(r"^(PORT|HOST)_(.+)$") + + +def render_template( + value: str, + *, + process_name: str, + world_index: int, + world_dir: Path, + port_plan: PortPlan, + configuration_addresses: dict[str, str], +) -> str: + """§2b's closed placeholder vocabulary. `preflight_bundle` has already validated every token + in `value` against this exact vocabulary before this ever runs, so a token this function + cannot resolve is an internal bug, not a bundle defect — raised as `ProcessRuntimeError`, + never one of `PreflightError`'s §2e codes (those are preflight's alone; see its own + docstring: "crosses the outbound seam").""" + + def resolve(match: re.Match[str]) -> str: + token = match.group(1) + if token == "WORLD_INDEX": + return str(world_index) + if token == "WORLD_DIR": + return str(world_dir) + if token == "DB_NAME": + return f"w{world_index}" + named = _NAMED_PLACEHOLDER.match(token) + if named: + kind, name = named.groups() + if kind == "HOST": + return "localhost" + try: + return str(port_plan.port_for(name, world_index)) + except KeyError: + raise ProcessRuntimeError( + "render", + "internal_unknown_placeholder", + f"{{{{{token}}}}} names {name!r}, which is not in this bundle's processes", + process=process_name, + ) from None + if token in configuration_addresses: + return configuration_addresses[token] + raise ProcessRuntimeError( + "render", + "internal_unknown_placeholder", + f"{{{{{token}}}}} has no resolution; preflight should have rejected this bundle", + process=process_name, + ) + + return _PLACEHOLDER.sub(resolve, value) + + +def render_environment( + process: SourceProcess, + *, + world_index: int, + world_dir: Path, + port_plan: PortPlan, + configuration_addresses: dict[str, str], +) -> dict[str, str]: + """`build_environment` is deliberately excluded — §2b: it "takes NO placeholders at all," so + it is merged raw by the env builders below, never passed through this renderer.""" + return { + key: render_template( + value, + process_name=process.name, + world_index=world_index, + world_dir=world_dir, + port_plan=port_plan, + configuration_addresses=configuration_addresses, + ) + for key, value in process.environment.items() + } + + +def select_process_secrets( + process: SourceProcess, *, secret_values: dict[str, str], secret_purposes: dict[str, str] +) -> dict[str, str]: + """§2b: "the provisioner injects every alias whose ref's `purpose` is listed [in + `secret_purposes`], under the **alias** as the env-var name." `secret_purposes` maps each + job-level alias to its `SecretRef.purpose` value — the same shape `process_preflight. + preflight_bundle`'s own `secret_refs` argument uses, so a caller that already ran preflight + has this for free. + + `SecretPurpose.SOURCE_CHECKOUT` is excluded unconditionally (F13, p5-round1-review): §1 states + it is "gateway-only; never uploaded to the guest," and preflight does not forbid a process + from legally *claiming* it (§2b's `secret_unclaimed`/`secret_missing` pair is scoped to + `target_provider` only) — the guest should not depend on the gateway alone never putting one + in `secrets.json` to keep that promise. + """ + claimed = { + purpose.value + for purpose in process.secret_purposes + if purpose is not SecretPurpose.SOURCE_CHECKOUT + } + return { + alias: value + for alias, value in secret_values.items() + if secret_purposes.get(alias) in claimed + } + + +# §2b enumerates exactly what a process receives: rendered `environment`, `build_environment`, +# purpose-matched secrets, and the PATH prepend below — the ambient `svc-control` environment is +# not on that list, and §1 marks the source untrusted (F12, p5-round1-review). A short, fixed +# allowlist of the interpreter/locale plumbing a process cannot run without, rather than +# `os.environ` wholesale — nothing here exports a secret today, but the entrypoint that will +# (a bearer token, a `FUTUREAGI_*` marker) is exactly the next thing wired on top of this module. +_INHERITED_ENV_ALLOWLIST = ("PATH", "HOME", "LANG", "TZ", "TMPDIR") + + +def _allowlisted_ambient_env(source: dict[str, str]) -> dict[str, str]: + env = {key: value for key, value in source.items() if key in _INHERITED_ENV_ALLOWLIST} + env.update({key: value for key, value in source.items() if key.startswith("LC_")}) + return env + + +def _base_process_env( + build_dir: Path, extra: dict[str, str] | None = None, *, base: dict[str, str] | None = None +) -> dict[str, str]: + """§2b: `build_environment` "merged" env, plus the provisioner's own unconditional PATH + prepend — applied last so it always wins even if `extra` (or the ambient environment) sets + its own `PATH`, for both build and run per the contract's own words. `base` defaults to an + allowlisted slice of the ambient environment (F12), not `os.environ` wholesale.""" + env = dict(base if base is not None else _allowlisted_ambient_env(os.environ)) + if extra: + env.update(extra) + prepend = [str(build_dir / ".venv" / "bin"), str(build_dir / "node_modules" / ".bin")] + # F14, p5-round1-review: `filter(None, ...)` — an unset/empty `PATH` would otherwise leave a + # trailing `:`, and POSIX `execvp` reads an empty PATH element as "current directory." cwd for + # both build and run is the customer's own build tree, so that would let a repo shipping an + # executable literally named `ls`/`git`/`sh` shadow the real one for any bare-name argv[0]. + env["PATH"] = os.pathsep.join(filter(None, [*prepend, env.get("PATH", "")])) + return env + + +# --- §2b copy-based build trees ----------------------------------------------------------------- + +# §0 (v1.7): "a repo needing an interpreter the snapshot lacks fails at BUILD time... reported +# `runtime_unsupported`, naming what the snapshot ships." Detection surface: an exec of a build +# step's argv[0] that looks like an interpreter name (`python`, `python3`, `python3.11`, `node`, +# `node20`, ...) raising `FileNotFoundError` — anything else that fails to exec, or exits +# nonzero, is `build_failed`. This is a judgement call the contract states as a rule +# ("the build step's failure is reported `runtime_unsupported`") without naming a detection +# mechanism; §0 also promises "python 3.11 and 3.12, node 20 and 22" specifically, so the pattern +# is scoped to those two families rather than every possible interpreter name. +_INTERPRETER_PATTERN = re.compile(r"^(python\d*(\.\d+)?|node\d*)(\.exe)?$") + + +def _looks_like_missing_interpreter(argv0: str) -> bool: + return bool(_INTERPRETER_PATTERN.match(Path(argv0).name)) + + +# --- process identity: user resolution and path containment (F1/F3, p5-round1-review) --------- + + +def default_user_resolver(username: str) -> "pwd.struct_passwd | None": + """§0: the base snapshot guarantees `svc-agent`/`svc-tools`/`svc-data` exist — on the hosted + path this always resolves. A dev box has none of them; returning `None` rather than raising + is what lets `_resolve_process_user`'s local-lane fallback work without a second seam.""" + try: + return pwd.getpwnam(username) + except KeyError: + return None + + +def _resolve_process_user( + user: ProcessUser, + *, + resolver: Callable[[str], "pwd.struct_passwd | None"], + require: bool, + process_name: str, + stage: str, + domain: FailureDomain, +) -> "pwd.struct_passwd | None": + """F1, p5-round1-review: every build tree and every spawned process must run under its + declared `user`, not the harness's own `svc-control` — otherwise an untrusted `agent` process + runs with read access to the whole-attempt capabilities bearer (§0 step 4) and, since every + process then shares one uid, mutual `/proc//environ` visibility into every other + process's injected secrets regardless of `secret_purposes`. + + `require=False` (the default, the local test lane's shape — no `svc-*` accounts on a dev box) + logs and returns `None`, so the caller runs unprivileged rather than failing every local run. + `require=True` is for a caller that knows it is on the hosted path, where the snapshot's own + guarantee means resolution failing is itself a typed failure. `domain` is the caller's own + §2f `spawn_failed` split (managed vs source) — this function has no process-kind knowledge of + its own, so it never guesses it. + """ + resolved = resolver(user.value) + if resolved is None: + if require: + raise ProcessRuntimeError( + stage, "spawn_failed", + f"{user.value!r} has no passwd entry; the hosted snapshot must guarantee it", + process=process_name, domain=domain, + ) + logger.warning( + "process %s declares user=%s but it is not resolvable on this host; running " + "unprivileged (local test lane fallback, not the hosted path)", + process_name, user.value, + ) + return resolved + + +def _default_chown(path: Path, uid: int, gid: int) -> None: + # `follow_symlinks=False`: F2 copies a within-tree symlink as a link, never dereferenced — + # chowning through it would touch whatever it points at instead of the link itself. + os.chown(path, uid, gid, follow_symlinks=False) + + +def _chown_tree(root: Path, *, uid: int, gid: int, chown: Callable[[Path, int, int], None]) -> None: + chown(root, uid, gid) + for dirpath, dirnames, filenames in os.walk(root): + for name in (*dirnames, *filenames): + chown(Path(dirpath) / name, uid, gid) + + +def _ensure_within(path: Path, root: Path, *, process_name: str, stage: str) -> Path: + """Defense in depth for F3, p5-round1-review. The model layer now constrains a process `name` + to `^[a-z0-9][a-z0-9_-]*$` (`bundle_v2.py`), which already makes a `/`, `..`, or absolute name + unspellable — a process named `/etc` used to path-join into `build_dir = Path("/etc")` + directly, which the very next line then `rmtree`'d as `svc-control`. This is the backstop for + anything that reaches a name-derived directory without going through that validator, checked + right before it is handed to `rmtree`/`mkdir`. Returns the ORIGINAL `path`, not + `path.resolve()`'d — this is a containment check, not a normalization; returning the resolved + form would silently rewrite every caller's path (e.g. through a `/tmp` -> `/private/tmp` + symlink on macOS) for no reason connected to the check itself. + """ + resolved = path.resolve() + resolved_root = root.resolve() + if not resolved.is_relative_to(resolved_root): + raise ProcessRuntimeError( + stage, "process_name_invalid", + f"{process_name!r} resolves to {resolved}, which escapes {resolved_root}", + process=process_name, + ) + return path + + +def _reject_escaping_symlinks(tree_root: Path, allowed_root: Path, *, process_name: str) -> None: + """F2, p5-round1-review: §2e item 2's symlink rejection is scoped to the bundle's OWN files — + nothing scans `/work/source` for a symlink pointing outside it, since the bundle "does NOT + embed the repository source" (§2 preamble). Left unchecked, a repo can ship + `config/creds.json -> /run/futureagi/capabilities.json` and have the provisioner (running as + svc-control until F1's privilege drop even applies) materialize that 0600 file's bytes into + the customer's own build tree. Walked against the SOURCE tree before anything is copied, so a + rejection never partially materializes a tree first. `.resolve()` (not raw `os.readlink`) + deliberately, so a relative target or a multi-hop symlink chain is followed all the way to + where it actually lands, not just its first hop. + """ + for entry in tree_root.rglob("*"): + if not entry.is_symlink(): + continue + target = entry.resolve() + if not target.is_relative_to(allowed_root): + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{entry.relative_to(tree_root)} is a symlink to {target}, which escapes " + "/work/source", + process=process_name, domain=FailureDomain.ENVIRONMENT, + ) + + +def _copytree_preserving_symlinks(src: Path, dst: Path) -> None: + # F2: `symlinks=True` — copy a link as a link rather than dereferencing it. `shutil. + # copytree`'s default (`symlinks=False`) follows every symlink and copies the TARGET's + # contents as a regular file, which is what let a symlink out of the checkout read as + # svc-control in the first place. A link that escaped `/work/source` was already rejected by + # `_reject_escaping_symlinks` before this ever runs; one that stays inside the tree is copied + # as-is and simply works (or dangles harmlessly) under the chowned, unprivileged build tree. + shutil.copytree(src, dst, symlinks=True) + + +_DEFAULT_BUILD_STEP_TIMEOUT_SECONDS = 600.0 + + +def build_process_tree( + process: SourceProcess, + *, + source_root: Path, + build_root: Path, + run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + copy: Callable[[Path, Path], None] | None = None, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS, +) -> Path: + """§2b: copy `source_root/` to `build_root//`, chowned to the + process's `user` (F1), then argv-exec each `build_commands` step there under that same user — + no shell, so `&&`/`$VAR`/globs/pipes do not work, which is why the model layer + (`bundle_v2.SourceProcess._shape`) already rejects an empty step. Runs once; the caller is + responsible for calling this exactly once per job, per process — this function itself has no + per-job memory. Raises on the first failing step (`ProcessRuntimeError`, stage="build"); never + partially succeeds silently past a failure. + """ + build_dir = build_root / process.name + _ensure_within(build_dir, build_root, process_name=process.name, stage="build") + + source_dir = source_root / process.working_directory + resolved_source_root = source_root.resolve() + resolved_source_dir = source_dir.resolve() + if not resolved_source_dir.is_relative_to(resolved_source_root): + # F2: a symlinked PATH COMPONENT (e.g. `/work/source/services` itself a symlink to `/`) + # used to be silently followed by the old bare `.resolve()` — `working_directory` passes + # the model layer's `_safe_relative` (no `..`, not absolute) without ever naming the + # escape. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{process.working_directory} resolves outside /work/source", + process=process.name, domain=FailureDomain.ENVIRONMENT, + ) + if not resolved_source_dir.is_dir(): + # F5, p5-round1-review: named explicitly, before any copy attempt, rather than letting + # `copytree`'s own `FileNotFoundError`/`NotADirectoryError` climb out untyped. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{process.working_directory} is absent or not a directory in the checkout", + process=process.name, domain=FailureDomain.ENVIRONMENT, + ) + _reject_escaping_symlinks(resolved_source_dir, resolved_source_root, process_name=process.name) + + if build_dir.exists(): + shutil.rmtree(build_dir) + try: + (copy or _copytree_preserving_symlinks)(resolved_source_dir, build_dir) + except (OSError, shutil.Error) as exc: + # F5: `PermissionError` on an unreadable subtree, `shutil.Error`'s aggregated failure on a + # dangling symlink — every copy-phase failure is the same deterministic, non-retryable + # fault as a missing directory, so it gets the same code. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"copying {process.working_directory}: {exc}", process=process.name, + domain=FailureDomain.ENVIRONMENT, + ) from exc + + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="build", domain=FailureDomain.AGENT, + ) + if resolved_user is not None: + _chown_tree(build_dir, uid=resolved_user.pw_uid, gid=resolved_user.pw_gid, chown=chown) + spawn_uid = resolved_user.pw_uid if resolved_user is not None else None + spawn_gid = resolved_user.pw_gid if resolved_user is not None else None + + env = _base_process_env(build_dir, process.build_environment) + for step in process.build_commands: + try: + result = run( + step, cwd=build_dir, env=env, capture_output=True, text=True, + timeout=build_step_timeout_seconds, user=spawn_uid, group=spawn_gid, + ) + except subprocess.TimeoutExpired as exc: + # F15, p5-round1-review: an install step wedged on a private registry with no DNS + # answer used to block the provisioner forever — the only backstop was the gateway's + # whole-job TTL, which arrives as SIGTERM to the entrypoint while this call is still + # inside an uninterruptible `subprocess.run`. + raise ProcessRuntimeError( + "build", "build_failed", + f"{step!r} exceeded the {build_step_timeout_seconds}s build-step timeout", + process=process.name, domain=FailureDomain.AGENT, + ) from exc + except FileNotFoundError as exc: + if not build_dir.is_dir(): + # JC1 ruling (p5-round1-review): a vanished build tree plus a `python*`/`node*` + # step raises the exact same `FileNotFoundError` as a missing interpreter — this + # disambiguates a filesystem fault from an interpreter-availability one before the + # argv[0] heuristic below ever gets a say. + raise ProcessRuntimeError( + "build", "source_tree_unavailable", + f"{build_dir} vanished before {step!r} could run", process=process.name, + domain=FailureDomain.ENVIRONMENT, + ) from exc + if _looks_like_missing_interpreter(step[0]): + raise ProcessRuntimeError( + "build", + "runtime_unsupported", + f"{step[0]!r} is not on the snapshot's PATH; the snapshot ships python " + "3.11/3.12 and node 20/22 only", + process=process.name, domain=FailureDomain.ENVIRONMENT, + ) from exc + raise ProcessRuntimeError( + "build", "build_failed", f"{step!r}: {exc}", process=process.name, + domain=FailureDomain.AGENT, + ) from exc + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "build", + "build_failed", + f"{step!r} exited {result.returncode}" + (f": {stderr}" if stderr else ""), + process=process.name, domain=FailureDomain.AGENT, + ) + return build_dir + + +# --- process spawn ------------------------------------------------------------------------------- + + +class SpawnedProcess(Protocol): + """What both a real `subprocess.Popen` wrapper and a test fake must provide — enough for + `depends_on`'s `log_marker` variant and for cleanup, nothing engine-specific. + + M7, p6-review-r1: `terminate()` alone only SENDS a signal — it does not wait for the process + to actually exit, which several callers used to assume by running a `copytree`/`rmtree`/port- + reuse on the very next line. `wait()`/`kill()` let `_terminate_and_wait` (below) make that + wait real, with a hard escalation for a process that ignores the polite signal. + """ + + def is_running(self) -> bool: ... + + def captured_output(self) -> str: ... + + def terminate(self) -> None: ... + + def interrupt(self) -> None: + """N12, p6-review-r2: SIGINT — postgres's own FAST shutdown (`pg_ctl -m fast`): rolls + back in-flight transactions and disconnects clients immediately, unlike `terminate()`'s + SIGTERM, which postgres treats as a SMART shutdown that waits for every client to + disconnect on its own. `_terminate_and_wait` prefers this for a postgres handle so tearing + one down while its own dependents still hold connections does not have to eat the full + kill escalation every time. + """ + ... + + def wait(self, timeout: float) -> bool: + """Blocks up to `timeout` seconds for the process to exit. Returns whether it did.""" + ... + + def kill(self) -> None: ... + + +class ProcessRunner(Protocol): + def __call__( + self, argv: Sequence[str], *, cwd: Path, env: dict[str, str], log_path: Path, + user: int | None = None, group: int | None = None, + ) -> SpawnedProcess: ... + + +class CapabilityProber(Protocol): + def __call__( + self, *, protocol: CapabilityProtocol, host: str, port: int, path: str | None, + user: str | None = None, password: str | None = None, dbname: str | None = None, + ) -> bool: ... + + +@dataclass +class PopenProcess: + """`SpawnedProcess` backed by a real subprocess. stdout+stderr are redirected to a log file + under the process's own scratch directory rather than captured through a pipe-reading thread + — simpler, and `started_check`'s `log_marker` only ever needs to re-read the file's current + contents, not race a live stream.""" + + popen: subprocess.Popen + log_path: Path + + def is_running(self) -> bool: + return self.popen.poll() is None + + def captured_output(self) -> str: + try: + return self.log_path.read_text(encoding="utf-8", errors="replace") + except OSError: + return "" + + def terminate(self) -> None: + self.popen.terminate() + + def interrupt(self) -> None: + self.popen.send_signal(signal.SIGINT) + + def wait(self, timeout: float) -> bool: + try: + self.popen.wait(timeout=timeout) + return True + except subprocess.TimeoutExpired: + return False + + def kill(self) -> None: + self.popen.kill() + + +_TERMINATE_WAIT_SECONDS = 5.0 +# Q12, p6-review-r2/-r3: rabbitmq's own broker shutdown routinely exceeds the 5s default — used +# only for the pre-`datadir_copy`-snapshot terminate, where a SIGKILL mid-mnesia-write would seal +# a corrupt baseline every world then restores from. R5, p6-review-r4: not a 30s ceiling — a +# wedged broker costs this value TWICE (`_terminate_and_wait` re-applies it to the post-kill() +# reap wait too), so the worst case at that call site is 60s. +_RABBITMQ_TERMINATE_WAIT_SECONDS = 30.0 + + +def _terminate_and_wait( + handle: SpawnedProcess, *, timeout: float = _TERMINATE_WAIT_SECONDS, + prefer_interrupt: bool = False, +) -> None: + """M7, p6-review-r1: `terminate()` alone only sends SIGTERM — postgres treats it as a SMART + shutdown that may not have even started, let alone finished, so the very next line at several + call sites (a `copytree` baseline snapshot, an `rmtree` before a port is reused) used to run + against a data directory a still-live server could still be writing to, or tried to rebind a + port the dying process had not yet released. Escalates to `kill()` only once `timeout` has + passed with no exit — a real SIGKILL, not a repeated SIGTERM, since a process that ignored the + first one is not going to notice a second. Never raises: a handle that is already gone, or + whose OS-level terminate/kill/wait itself errors, must not block whatever cleanup is calling + this (`close()`'s own idempotency, m10, is what actually made this matter). + + N12, p6-review-r2: `prefer_interrupt` swaps the FIRST signal from SIGTERM to SIGINT (postgres's + fast shutdown) — the caller decides this per-engine, since this function has no engine + knowledge of its own. + """ + try: + if prefer_interrupt: + handle.interrupt() + else: + handle.terminate() + except OSError: + pass + if handle.wait(timeout): + return + try: + handle.kill() + except OSError: + pass + # N13, p6-review-r2: the return here used to be discarded — a child still not reaped after + # SIGKILL (a wedged kernel wait, not something any further signal can fix) left no trace + # anywhere. Nothing past SIGKILL to escalate TO, but silently swallowing that fact is strictly + # worse than a log line a caller could act on (flag the sandbox instead of assuming cleanup + # succeeded). + if not handle.wait(timeout): + logger.warning("_terminate_and_wait: process did not exit even after kill()") + + +def _prefers_interrupt(manifest: EnvironmentBundleV2, process_name: str) -> bool: + """N12, p6-review-r2: postgres is the one engine whose fast-shutdown signal genuinely differs + from a plain SIGTERM (see `SpawnedProcess.interrupt`) — every other process (`source` or + redis/rabbitmq) just gets the ordinary `terminate()` path.""" + processes_by_name = {process.name: process for process in manifest.processes} + process = processes_by_name.get(process_name) + return isinstance(process, ManagedProcess) and process.engine is ManagedEngine.POSTGRES + + +def _default_log_chown(path: Path, uid: int, gid: int) -> None: + os.chown(path, uid, gid) + + +def default_process_runner( + argv: Sequence[str], *, cwd: Path, env: dict[str, str], log_path: Path, + user: int | None = None, group: int | None = None, + chown: Callable[[Path, int, int], None] = _default_log_chown, +) -> PopenProcess: + log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = log_path.open("wb") + if user is not None: + # F1, p5-round1-review: the log file is created by the harness (svc-control) before the + # child's privilege drop — the child can still WRITE through the inherited fd regardless + # of on-disk mode (permission checks happen at open(), not at write()), but leaving it + # svc-control-owned means nothing running as the child's own user could ever open it + # fresh afterward. Chowned to match, not just handed over unchecked. `chown` is injectable + # like every other chown call in this module (`os.chown` to a foreign uid needs root/ + # CAP_CHOWN — the same privilege `user=`/`group=` below already requires, so a real + # hosted deployment has both or neither; a dev-box test fakes it structurally). + chown(log_path, user, group if group is not None else -1) + popen = subprocess.Popen( + list(argv), cwd=cwd, env=env, stdout=log_file, stderr=subprocess.STDOUT, + user=user, group=group, + ) + return PopenProcess(popen=popen, log_path=log_path) + + +@dataclass +class SpawnedWorldProcess: + process_name: str + handle: SpawnedProcess + port: int + world_index: int | None # None for a job-shared managed engine + # B2, p6-review-r1: the resolved spawn identity, carried on the handle so a caller that needs + # to run something ELSE (a seed file) as this same process's user does not have to re-resolve + # it a second time. `None` in the local-lane fallback, same as `spawn_uid`/`spawn_gid` above. + uid: int | None = None + gid: int | None = None + # The voice dispatch identity (LIVEKIT_AGENT_NAME) exists only in the process's rendered + # per-world environment, which is discarded after spawn — carried here so the provider can + # surface it on the world's runtime metadata without re-rendering templates. + dispatch_agent_name: str | None = None + + +def _dispatch_metadata( + handles: "dict[str, SpawnedWorldProcess]", +) -> dict[str, "JsonValue"]: + """A world with exactly one voice agent gets its dispatch identity on the runtime metadata; + anything else leaves the key absent so the call runner's own typed pre-dial failure fires + instead of a call being dialed at an arbitrarily chosen agent. Ambiguity is loud here + because the pre-dial message cannot say WHY the key is missing.""" + names = sorted({ + name + for handle in handles.values() + if (name := (handle.dispatch_agent_name or "").strip()) + }) + if len(names) == 1: + return {"livekit_agent_name": names[0]} + if len(names) > 1: + logger.warning( + "world declares %d distinct LIVEKIT_AGENT_NAME values (%s); " + "leaving dispatch identity unset", + len(names), ", ".join(names), + ) + return {} + + +# --- managed-engine launch commands --------------------------------------------------------- +# +# §2b fixes the catalog (engine/version/role/db-name/strategies) and the port formula; it does +# not fix an exact launch invocation for any of the three engines — that is provisioner-internal, +# same status as `LocalComposeRuntimeProvider`'s own Compose mechanics. The commands below are a +# defensible V1 default, kept swappable through `ProcessRunner`/`sync_run` injection; they are +# never executed in this module's own test lane (§0: assumed on PATH, never required in tests). + + +def postgres_bootstrap_argv( + *, data_dir: Path, credentials: EngineCredentials, pwfile: Path +) -> list[str]: + return [ + "initdb", "-D", str(data_dir), "-U", credentials.username, + "--pwfile", str(pwfile), "-A", "scram-sha-256", + ] + + +def postgres_daemon_argv(*, data_dir: Path, port: int) -> list[str]: + # n1, p6-review-r1: every connection this module makes is TCP `-h localhost` (seed, sentinel, + # canary, probes) — nothing ever dials the unix socket, so `-k ""` closes that listening + # surface entirely instead of leaving it open at `data_dir` unused. + return ["postgres", "-D", str(data_dir), "-p", str(port), "-k", "", "-h", "localhost"] + + +def redis_daemon_argv(*, data_dir: Path, port: int) -> list[str]: + return [ + "redis-server", "--port", str(port), "--dir", str(data_dir), + "--daemonize", "no", "--save", "", + ] + + +def rabbitmq_daemon_argv() -> list[str]: + return ["rabbitmq-server"] + + +def rabbitmq_enabled_plugins_text() -> str: + """M8, p6-review-r1: the Erlang term-list format `rabbitmq-server` reads at boot to decide + which plugins are on. `rabbitmq_daemon_env` sets node/auth/data-dir env vars only — nothing + turns the management app (the HTTP API `rabbitmqadmin` seeding and the sentinel/canary queue- + depth reads both depend on) on; without this file it is off and every one of those calls + connection-refuses.""" + return "[rabbitmq_management].\n" + + +def rabbitmq_conf_text(*, management_port: int, credentials: EngineCredentials) -> str: + """N5, p6-review-r2 (MAJOR): a BARE `rabbitmq-server` (§0: no Docker in the sandbox) reads + `default_user`/`default_pass` from THIS file — `RABBITMQ_DEFAULT_USER`/`RABBITMQ_DEFAULT_PASS` + (`rabbitmq_daemon_env`, below) are a convention the official Docker image's ENTRYPOINT + translates onto these same keys; nothing performs that translation for a directly-`exec`'d + `rabbitmq-server`, so without this the node initializes with the built-in `guest` account + instead of the catalog's declared `harness` role (§2b) — every rabbitmq call this module makes + (seed, sentinel, canary) authenticates as `harness` and would 401 against a node that never + heard of it. Q4, p6-review-r3: no `loopback_users` line here — `loopback_users = none` does + the OPPOSITE of what a prior version of this docstring claimed (it widens `guest`'s reach to + the whole network, RabbitMQ's own default `[guest]` restricts it to loopback already); with + `default_user = harness` a fresh node never creates a `guest` account at all, so the setting + buys nothing either way and is left out rather than left wrong. + + Also pins the management HTTP listener to this module's own `+10000`-offset port + (`_rabbitmq_management_port`) — the plugin's own built-in default is 15672, which sits inside + §2b's per-world process port band `[15000,15799]` and would alias an allocated process port; + binding `127.0.0.1` matches every other engine, which is localhost-only in V1. + """ + return ( + f"default_user = {credentials.username}\n" + f"default_pass = {credentials.password}\n" + f"management.tcp.port = {management_port}\n" + "management.tcp.ip = 127.0.0.1\n" + ) + + +def rabbitmq_daemon_env( + *, data_dir: Path, port: int, credentials: EngineCredentials +) -> dict[str, str]: + """`RABBITMQ_DEFAULT_USER`/`RABBITMQ_DEFAULT_PASS` are kept here as harmless redundancy (they + are exactly what the Docker image's entrypoint would consume, if the snapshot ever turned out + to wrap one after all — N5's own open question for the snapshot owner) — `rabbitmq_conf_text` + above is the credential source a BARE server actually reads. + + Q2, p6-review-r3 (MAJOR); R2, p6-review-r4: both `RABBITMQ_MNESIA_DIR` (a fixed, node-name-free + path) and `RABBITMQ_MNESIA_BASE` (`data_dir` itself) are set. `MNESIA_DIR` wins for the data + path, so a `datadir_copy` baseline snapshotted at world 0's port no longer sits under a path + only world 0's port-derived node name would look in. `MNESIA_BASE` still has to stay set: + rabbitmq derives OTHER paths from the base, not the dir — `RABBITMQ_PLUGINS_EXPAND_DIR` among + them — so dropping it moves those outside `data_dir`, to the installation default the spawn + user cannot write, and every rabbitmq spawn fails to boot (bootstrap included). + + KNOWN DEFECT (R1, p6-review-r4 — disclosed, not redesigned this phase): the mnesia DATA PATH + above is node-name-free, but the mnesia SCHEMA on disk is not — `schema.DAT` and rabbitmq's own + cluster-status file both record the node name (`RABBITMQ_NODENAME` below) the tables were + created under, and per-world node names must stay distinct (every world's broker registers with + the same epmd in the same sandbox). A `datadir_copy` baseline therefore boots cleanly in world 0 + only; every other world points its node at a directory holding world 0's node-bound schema, + which the broker will not adopt — expect a boot/readiness failure surfacing as + `depends_on_timeout` from `_wait_for_store_ready`, not a silent empty node. Recorded + follow-up: replace the datadir copy with a definitions-export/import restore through the + existing `default_rabbitmq_definitions_importer` seam, which is per-world safe by + construction.""" + return { + "RABBITMQ_NODE_PORT": str(port), + "RABBITMQ_MNESIA_DIR": str(data_dir / "mnesia"), + "RABBITMQ_MNESIA_BASE": str(data_dir), + "RABBITMQ_LOG_BASE": str(data_dir), + "RABBITMQ_DEFAULT_USER": credentials.username, + "RABBITMQ_DEFAULT_PASS": credentials.password, + "RABBITMQ_NODENAME": f"harness-{port}@localhost", + } + + +def spawn_managed_process( + process: ManagedProcess, + *, + port: int, + data_dir: Path, + credentials: EngineCredentials | None, + runner: ProcessRunner = default_process_runner, + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, +) -> SpawnedWorldProcess: + """Spawns one managed-engine daemon. Postgres alone needs a one-time synchronous bootstrap + (`initdb`) before the daemon can start at all — run through `sync_run`, not `runner`, since + it must complete before the daemon exec happens, not run alongside it. Redis/RabbitMQ + initialize their own data directory on first boot and need no separate step. + + `data_dir` is chowned to the engine's declared `user` (`svc-data`, per §2b) before anything + runs in it (F1, p5-round1-review) — otherwise a daemon spawned under that user could not even + write its own data directory, which the harness (running as `svc-control`) created. + """ + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="spawn", domain=FailureDomain.INFRASTRUCTURE, + ) + try: + data_dir.mkdir(parents=True, exist_ok=True) + # postgres demands mode 0700 on the directory itself regardless of engine; harmless for + # redis/rabbitmq, so applied uniformly rather than special-cased per engine. N14, + # p6-review-r2: BEFORE the chown below, not after — `svc-control` still owns `data_dir` at + # this point and the chmod is free; once ownership has moved to the engine's own uid, the + # SAME call would raise `PermissionError` for a `svc-control` that can chown but is not + # itself root. + data_dir.chmod(0o700) + if resolved_user is not None: + # B6, p6-review-r1: a single top-level `chown` was fine for a FRESH, empty data_dir + # (the daemon's own first-boot `initdb` then creates everything else as `spawn_uid` + # itself) but wrong the moment this same call runs against a data_dir a COPY just + # populated (`freeze_baseline`'s snapshot restore, `_seal_world_store`'s `datadir_copy` + # reseal) — the copy runs as the provisioner, so every file underneath stayed + # provisioner-owned and postgres refuses to start against a data directory it does not + # fully own. + _chown_tree(data_dir, uid=resolved_user.pw_uid, gid=resolved_user.pw_gid, chown=chown) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2: §4.6 — "engine/process/filesystem failures during provisioning -> + # infrastructure." mkdir/chmod/chown on this engine's own data directory used to raise + # bare, giving `provision()`'s caller nothing to map; every failure this module's own + # data-dir setup can hit is the same infrastructure class B5 already typed for the store + # seams themselves. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + spawn_uid = resolved_user.pw_uid if resolved_user is not None else None + spawn_gid = resolved_user.pw_gid if resolved_user is not None else None + + env = _allowlisted_ambient_env(os.environ) + if process.engine is ManagedEngine.POSTGRES: + if credentials is None: + raise ProcessRuntimeError( + "spawn", "spawn_failed", "postgres requires generated credentials", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) + if not (data_dir / "PG_VERSION").exists(): + pwfile = data_dir.parent / f".{process.name}.pwfile" + # F7, p5-round1-review: created at 0600 ATOMICALLY. `write_text` then `chmod` creates + # the file at `0666 & ~umask` (typically 0644) and only narrows it afterward — a + # classic create-then-chmod TOCTOU that leaves the generated superuser password + # world-readable for the window in between. `O_EXCL` additionally refuses to follow a + # pre-planted symlink/file already sitting at this path. + fd = os.open(pwfile, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(credentials.password + "\n") + if resolved_user is not None: + # `initdb` itself runs as `resolved_user` below (via `sync_run`'s `user=`) — + # it must be able to READ its own 0600 file, so ownership follows the same + # user, never the mode widening to let anyone else read it too. + chown(pwfile, resolved_user.pw_uid, resolved_user.pw_gid) + bootstrap_argv = postgres_bootstrap_argv( + data_dir=data_dir, credentials=credentials, pwfile=pwfile + ) + result = sync_run( + bootstrap_argv, capture_output=True, text=True, + user=spawn_uid, group=spawn_gid, + ) + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "spawn", "spawn_failed", + f"initdb exited {result.returncode}: {stderr}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) + finally: + pwfile.unlink(missing_ok=True) + argv = postgres_daemon_argv(data_dir=data_dir, port=port) + elif process.engine is ManagedEngine.REDIS: + argv = redis_daemon_argv(data_dir=data_dir, port=port) + elif process.engine is ManagedEngine.RABBITMQ: + if credentials is None: + raise ProcessRuntimeError( + "spawn", "spawn_failed", "rabbitmq requires generated credentials", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) + # M8, p6-review-r1: written fresh on every spawn (job bootstrap AND every world's own + # instance) — cheap, and means a `datadir_copy` restore can never carry a stale plugin/ + # port config forward from whatever was on disk when the baseline was snapshotted. + plugins_path = data_dir / "enabled_plugins" + conf_path = data_dir / "rabbitmq.conf" + try: + plugins_path.write_text(rabbitmq_enabled_plugins_text(), encoding="utf-8") + # N5, p6-review-r2: 0600 on CREATE — this file carries the generated password in + # cleartext (`rabbitmq_conf_text`). Plain `O_TRUNC`, not `O_EXCL`: unlike the postgres + # pwfile, `data_dir` legitimately already HAS this file on a `datadir_copy` restore + # (it was copied in from the baseline snapshot along with everything else), and the + # mode set at CREATE time already carried through that copy (`shutil.copytree`'s + # default `copy2` preserves permission bits) — `O_EXCL` would just raise + # `FileExistsError` on every world after the first. + fd = os.open(conf_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(rabbitmq_conf_text( + management_port=_rabbitmq_management_port(port), credentials=credentials, + )) + if resolved_user is not None: + chown(plugins_path, resolved_user.pw_uid, resolved_user.pw_gid) + chown(conf_path, resolved_user.pw_uid, resolved_user.pw_gid) + except OSError as exc: + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: writing rabbitmq config: {exc}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + env.update(rabbitmq_daemon_env(data_dir=data_dir, port=port, credentials=credentials)) + env["RABBITMQ_ENABLED_PLUGINS_FILE"] = str(plugins_path) + # N6, p6-review-r2: the FULL path, extension included. Modern RabbitMQ (the catalog's + # 3.13) documents `RABBITMQ_CONFIG_FILE` carrying `.conf` itself and unambiguously accepts + # it that way — the pre-3.7 classic-config "the server appends .conf for you" behavior + # this used to depend on is not how the catalog's version works, and the failure was + # silent: the file on disk is `rabbitmq.conf`, the env var said `rabbitmq`, and if the + # append never happens the server falls back to defaults — including the bare management + # port 15672, which sits inside §2b's own per-world port band. + env["RABBITMQ_CONFIG_FILE"] = str(conf_path) + argv = rabbitmq_daemon_argv() + else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"unknown engine {process.engine!r}", process=process.name, + domain=FailureDomain.INFRASTRUCTURE, + ) + try: + handle = runner( + argv, cwd=data_dir, env=env, log_path=data_dir / "process.log", + user=spawn_uid, group=spawn_gid, + ) + except FileNotFoundError as exc: + raise ProcessRuntimeError( + "spawn", "spawn_failed", str(exc), process=process.name, + domain=FailureDomain.INFRASTRUCTURE, + ) from exc + return SpawnedWorldProcess( + process_name=process.name, handle=handle, port=port, world_index=None, + uid=spawn_uid, gid=spawn_gid, + ) + + +def spawn_source_process( + process: SourceProcess, + *, + build_dir: Path, + world_dir: Path, + world_index: int, + port_plan: PortPlan, + configuration_addresses: dict[str, str], + secret_values: dict[str, str], + secret_purposes: dict[str, str], + runner: ProcessRunner = default_process_runner, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + require_declared_user: bool = False, + chown: Callable[[Path, int, int], None] = _default_chown, +) -> SpawnedWorldProcess: + """§2b: `run_command` "exec'd once per world with cwd `/work/build//`" — the build tree + is read-only by convention at run time; `world_dir` (`{{WORLD_DIR}}`) is this process's own + per-world writable scratch, created here since nothing upstream of spawn needs it to exist. + Chowned to the process's declared `user` before spawn (F1, p5-round1-review) — otherwise a + process running as anyone but the harness's own uid could not write into its own `{{WORLD_ + DIR}}` at all, since the harness (as `svc-control`) is the one that just created it. + """ + resolved_user = _resolve_process_user( + process.user, resolver=user_resolver, require=require_declared_user, + process_name=process.name, stage="spawn", domain=FailureDomain.AGENT, + ) + try: + world_dir.mkdir(parents=True, exist_ok=True) + if resolved_user is not None: + chown(world_dir, resolved_user.pw_uid, resolved_user.pw_gid) + except OSError as exc: + # N9, p6-review-r2: same class as `spawn_managed_process`'s own data-dir setup boundary — + # a permission error creating/chowning this process's `{{WORLD_DIR}}` used to raise bare. + raise ProcessRuntimeError( + "spawn", "spawn_failed", f"{process.name}: preparing {world_dir}: {exc}", + process=process.name, domain=FailureDomain.AGENT, + ) from exc + rendered = render_environment( + process, + world_index=world_index, + world_dir=world_dir, + port_plan=port_plan, + configuration_addresses=configuration_addresses, + ) + injected = select_process_secrets( + process, secret_values=secret_values, secret_purposes=secret_purposes + ) + env = _base_process_env( + build_dir, {**(process.build_environment or {}), **rendered, **injected} + ) + try: + handle = runner( + list(process.run_command), cwd=build_dir, env=env, log_path=world_dir / "process.log", + user=resolved_user.pw_uid if resolved_user is not None else None, + group=resolved_user.pw_gid if resolved_user is not None else None, + ) + except FileNotFoundError as exc: + raise ProcessRuntimeError( + "spawn", "spawn_failed", str(exc), process=process.name, domain=FailureDomain.AGENT, + ) from exc + port = port_plan.port_for(process.name, world_index) + return SpawnedWorldProcess( + process_name=process.name, handle=handle, port=port, world_index=world_index, + uid=resolved_user.pw_uid if resolved_user is not None else None, + gid=resolved_user.pw_gid if resolved_user is not None else None, + dispatch_agent_name=rendered.get("LIVEKIT_AGENT_NAME") or None, + ) + + +# --- filesystem layout (§0's guaranteed paths, plus one this module must invent) -------------- + + +def build_tree_dir(work_directory: Path, process_name: str) -> Path: + return _ensure_within( + work_directory / "build" / process_name, work_directory, + process_name=process_name, stage="build", + ) + + +def world_scratch_dir(work_directory: Path, world_index: int, process_name: str) -> Path: + return _ensure_within( + work_directory / "worlds" / f"w{world_index}" / process_name, work_directory, + process_name=process_name, stage="spawn", + ) + + +def managed_engine_data_dir( + work_directory: Path, process_name: str, *, world_index: int | None +) -> Path: + """§0 fixes `/work/build//` and `/work/worlds/w//` but names no path for a + managed engine's own data directory. Per-world engines reuse the per-world scratch shape + (`world_index` given) so `reset`'s `datadir_copy` case (§4.2: "restore its data directory") + has an unambiguous per-world location. Job-shared engines (`world_index=None`) get a + job-level directory outside any single world's tree, since nothing about them is per-world — + `/work/managed//`, which §0 v1.8 added to the layout block (JC4, p5-round1-review). + """ + if world_index is None: + path = work_directory / "managed" / process_name + else: + return world_scratch_dir(work_directory, world_index, process_name) + return _ensure_within(path, work_directory, process_name=process_name, stage="spawn") + + +# --- §2b depends_on wait ------------------------------------------------------------------------- + + +def _readiness_probes_for_process( + manifest: EnvironmentBundleV2, process_name: str +) -> list[ReadinessProbeV2]: + """F8, p5-round1-review: returns EVERY declared probe backing this process, not just the + first-listed one — matching `healthy()`'s own definition of ready (§4 point 3: "declared + `readiness` probes," plural, unqualified). A process backing two capabilities used to be + treated as ready by `depends_on` the moment the first-listed probe passed, then immediately + reported unhealthy the instant `healthy()` ran, because the two functions disagreed about + what "ready" meant.""" + capability_slugs = { + slug + for slug, capability in manifest.capabilities.items() + if capability.service == process_name + } + return [probe for probe in manifest.readiness if probe.capability in capability_slugs] + + +def _tcp_probe(host: str, port: int, *, timeout: float = 0.75) -> bool: + try: + with socket.create_connection((host, port), timeout=timeout): + return True + except OSError: + return False + + +def _probe_http( + host: str, port: int, path: str | None, *, user: str | None = None, + password: str | None = None, timeout: float = 1.0, +) -> bool: + """N7, p6-review-r2: `user`/`password`, when both given, ride along as a Basic-auth header — + added for the rabbitmq management listener probe (`_wait_for_store_ready`), which otherwise + 401s on every attempt regardless of whether the listener is actually up.""" + url = f"http://{host}:{port}/{(path or '').lstrip('/')}" + try: + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + request = urllib.request.Request(url) + if user is not None and password is not None: + credential_bytes = f"{user}:{password}".encode("utf-8") + request.add_header( + "Authorization", "Basic " + base64.b64encode(credential_bytes).decode() + ) + with opener.open(request, timeout=timeout) as response: + response.read(256) + return 200 <= response.status < 400 + except Exception: + # Q6, p6-review-r3: `http.client.HTTPException` (a malformed status line, a truncated + # body) is not an `OSError` — a probe's contract is bool-not-raise, same broad form + # `_probe_postgres` already uses below. + return False + + +def _probe_postgres( + host: str, port: int, *, user: str | None = None, password: str | None = None, + dbname: str | None = None, timeout: float = 1.0, +) -> bool: + """F9, p5-round1-review: previously connected as `user="postgres", dbname="postgres"` — + neither exists, since the catalog's `initdb -U harness` creates only the `harness` role — and + treated almost any resulting `OperationalError` as "answered, therefore ready," including + `FATAL: the database system is starting up`, postgres's OWN response while still in recovery + (it binds its listen socket early). With real generated credentials threaded through, this + runs an actual `SELECT 1`, which a starting-up server cannot pass. Without credentials (no + generated role for this call site, or `psycopg` absent) it falls back to a bare TCP probe — + strictly weaker, but no longer claims false readiness from a substring match either. + """ + try: + import psycopg # type: ignore[import-not-found] + except ImportError: + return _tcp_probe(host, port, timeout=timeout) + if user is None or dbname is None: + return _tcp_probe(host, port, timeout=timeout) + try: + connection = psycopg.connect( + host=host, port=port, user=user, password=password, dbname=dbname, + connect_timeout=timeout, + ) + try: + connection.execute("SELECT 1") + finally: + connection.close() + return True + except psycopg.OperationalError: + return False + except Exception: + return False + + +def default_capability_prober( + *, protocol: CapabilityProtocol, host: str, port: int, path: str | None, + user: str | None = None, password: str | None = None, dbname: str | None = None, +) -> bool: + if protocol is CapabilityProtocol.POSTGRES: + return _probe_postgres(host, port, user=user, password=password, dbname=dbname) + if protocol is CapabilityProtocol.HTTP: + return _probe_http(host, port, path, user=user, password=password) + return _tcp_probe(host, port) + + +def _poll_until( + condition: Callable[[], bool], + *, + timeout: float, + interval: float, + clock: Callable[[], float], + sleep: Callable[[float], None], + timeout_error: Callable[[], Exception], +) -> None: + deadline = clock() + timeout + while True: + if condition(): + return + if clock() >= deadline: + raise timeout_error() + sleep(interval) + + +def _postgres_probe_credentials( + capability: CapabilityV2, + *, + world_index: int, + credentials: dict[str, EngineCredentials] | None, +) -> tuple[str | None, str | None, str | None]: + if capability.protocol is not CapabilityProtocol.POSTGRES: + return None, None, None + creds = (credentials or {}).get(capability.service) + if creds is None: + return None, None, None + return creds.username, creds.password, f"w{world_index}" + + +def wait_for_dependency( + manifest: EnvironmentBundleV2, + dependency_name: str, + *, + world_index: int, + port_plan: PortPlan, + spawned: SpawnedWorldProcess, + credentials: dict[str, EngineCredentials] | None = None, + prober: CapabilityProber = default_capability_prober, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """§2b: "the dependent starts only after the dependency's capability `readiness` probe passes + (or its `started_check`, or immediately after spawn if it has neither)." Priority order per + the contract's own parenthetical: a declared capability readiness probe first; `started_check` + only when the dependency backs no such capability (and only `SourceProcess` ever carries one — + `ManagedProcess` has no `started_check` field); otherwise return immediately, since spawn + already happened before this is ever called. + + A dependency backing more than one capability must pass EVERY declared probe (F8), matching + `healthy()`'s own definition — the combined timeout is the longest of them (no probe is cut + off early), the combined poll interval is the tightest of them (no probe is polled less often + than it declared). `credentials` (F9) lets a postgres probe run a real `SELECT 1` instead of + a bare TCP connect; omitted, it degrades to the same TCP-only check as before. + """ + probes = _readiness_probes_for_process(manifest, dependency_name) + if probes: + port = port_plan.port_for(dependency_name, world_index) + + def probe_ready() -> bool: + for probe in probes: + capability = manifest.capabilities[probe.capability] + user, password, dbname = _postgres_probe_credentials( + capability, world_index=world_index, credentials=credentials + ) + if not prober( + protocol=capability.protocol, host="localhost", port=port, path=probe.path, + user=user, password=password, dbname=dbname, + ): + return False + return True + + combined_timeout = max(probe.timeout_seconds for probe in probes) + _poll_until( + probe_ready, + timeout=combined_timeout, + interval=min(probe.interval_seconds for probe in probes), + clock=clock, + sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "depends_on", + "depends_on_timeout", + f"{dependency_name}: readiness probe did not pass within {combined_timeout}s", + process=dependency_name, domain=FailureDomain.INFRASTRUCTURE, + ), + ) + return + + processes_by_name = {process.name: process for process in manifest.processes} + dependency = processes_by_name[dependency_name] + started_check = dependency.started_check if isinstance(dependency, SourceProcess) else None + if started_check is None: + return # neither a readiness probe nor a started_check — ready immediately after spawn. + + if started_check.port: + # §2b (v1.8): the field only SELECTS the port-probe variant — the dialed port is always + # the dependency's own allocated one (`port_plan.port_for`, honoring `fixed_port`), never + # a literal read from the manifest (F4, p5-round1-review). A literal cannot be correct for + # more than one world, since the formula port differs by `world_index`. + port = port_plan.port_for(dependency_name, world_index) + + def condition() -> bool: + return _tcp_probe("localhost", port) + else: + marker = started_check.log_marker + + def condition() -> bool: + return marker in spawned.handle.captured_output() + + _poll_until( + condition, + timeout=started_check.timeout_seconds, + interval=0.25, + clock=clock, + sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "depends_on", + "depends_on_timeout", + f"{dependency_name}: started_check did not pass within " + f"{started_check.timeout_seconds}s", + process=dependency_name, domain=FailureDomain.INFRASTRUCTURE, + ), + ) + + +def _topological_order(manifest: EnvironmentBundleV2) -> list[str]: + """Dependencies before dependents. `preflight_bundle` (`_verify_depends_on`) already rejects + cycles and unknown names before this ever runs, so a DAG is assumed here, not re-verified.""" + graph = {process.name: list(process.depends_on) for process in manifest.processes} + order: list[str] = [] + visited: set[str] = set() + + def visit(name: str) -> None: + if name in visited: + return + visited.add(name) + for dependency_name in graph[name]: + visit(dependency_name) + order.append(name) + + for name in graph: + visit(name) + return order + + +# --- §2c/§5 store command seams ----------------------------------------------------------------- +# +# One injectable seam per engine, mirroring `ProcessRunner`/`CapabilityProber`: real production +# code talks to the actual store, a test hands in a spy. `SqlRunner` alone carries the "SQL spy" +# tests assert TEMPLATE/DROP/CREATE/sentinel statements against — postgres is the only engine +# `no_sql_store` (§2e item 6) guarantees, so it is the only one whose statement-level shape this +# module commits to; redis/rabbitmq get the same command-injection treatment but a plainer one +# (`argv`, not a statement AST), matching how much less this contract pins about them (§2c: seeded +# "only if the repo ships seed state for them"). + + +class SqlRunner(Protocol): + def __call__( + self, *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, + ) -> list[tuple[Any, ...]]: ... + + +def default_sql_runner( + *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, +) -> list[tuple[Any, ...]]: + """psycopg-backed, import-guarded like `_probe_postgres` — never exercised in this module's + own test lane (no real postgres here), always injected by a test's SQL spy instead. + + N8, p6-review-r2 (MAJOR): `read_only`, when set, puts the session into a read-only default + transaction mode before running `statement` — the ONLY thing that made `sentinel.query` (and + the baseline row-count reads it shares this code path with) different from an arbitrary write + was convention: it runs as `harness`, the role `initdb -U harness` makes the postgres + SUPERUSER, over a plain autocommit session, executing customer-authored content from an + untrusted repo verbatim. `world-handle-interface.md` v3.4's own `query()` runs on a `SET + TRANSACTION READ ONLY` connection and calls that "the guard" — this is the provisioner's side + of the same guard, for every call site that is supposed to be a read (`_call_sql`'s own + `read_only=True` callers). + """ + import psycopg # type: ignore[import-not-found] + + with psycopg.connect( + host=host, port=port, user=user, password=password, dbname=dbname, autocommit=True, + ) as connection: + if read_only: + connection.execute("SET default_transaction_read_only = on") + cursor = connection.execute(statement) + try: + return cursor.fetchall() + except psycopg.ProgrammingError: + return [] # a DDL/utility statement (CREATE DATABASE, ALTER DATABASE, ...) has no rows. + + +class RedisCommandRunner(Protocol): + def __call__(self, *, host: str, port: int, command: Sequence[str]) -> Any: ... + + +def default_redis_command_runner(*, host: str, port: int, command: Sequence[str]) -> Any: + import redis # type: ignore[import-not-found] + + client = redis.Redis(host=host, port=port) + try: + return client.execute_command(*command) + finally: + client.close() + + +class RabbitmqQueueInspector(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str + ) -> int: ... + + +class RabbitmqQueueDeclarer(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str, message: str, + ) -> None: ... + + +class RabbitmqQueueDeleter(Protocol): + """Q10, p6-review-r3: no remaining caller — N15 (p6-review-r2) moved canary cleanup to the + reset-from-baseline path this seam predates. Retained deliberately (constructor param, + `SpawnContext` field) as the explicit-delete seam a future canary strategy may still want, + rather than ripped out and re-added from scratch.""" + + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, queue: str, + ) -> None: ... + + +# --- B5, p6-review-r1: typed failures at the store-command seams ----------------------------- +# +# Every call below runs AFTER the store's own readiness has already been established (B4's wait, +# or an already-running job-shared engine) — a driver exception here is the store rejecting or +# erroring on the provisioner's OWN statement, never a connection race, so it is §2f's +# `store_statement_failed` (infrastructure, retryable — v1.10), never `depends_on_timeout` (that +# code belongs to the readiness WAIT itself, which `default_capability_prober` already degrades +# to a bool instead of raising) and never `seed_failed` (reserved for the customer's own +# migration/seed CONTENT, applied through `sync_run`'s exit code, not through these seams at +# all). One wrapper per seam, matching the existing one-Protocol-per-engine-concern split. + + +def _call_sql( + sql_runner: SqlRunner, *, stage: str, process_name: str, host: str, port: int, user: str, + password: str, dbname: str, statement: str, read_only: bool = False, +) -> list[tuple[Any, ...]]: + try: + return sql_runner( + host=host, port=port, user=user, password=password, dbname=dbname, statement=statement, + read_only=read_only, + ) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued statement: {exc}", + process=process_name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + + +def _call_redis( + redis_runner: RedisCommandRunner, *, stage: str, process_name: str, host: str, port: int, + command: Sequence[str], +) -> Any: + try: + return redis_runner(host=host, port=port, command=command) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued command: {exc}", + process=process_name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + + +def _call_rabbitmq( + rabbitmq_inspector: RabbitmqQueueInspector, *, stage: str, process_name: str, host: str, + port: int, credentials: EngineCredentials, queue: str, +) -> int: + try: + return rabbitmq_inspector(host=host, port=port, credentials=credentials, queue=queue) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected a provisioner-issued queue inspection: {exc}", + process=process_name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + + +_RABBITMQ_DEPTH_READ_ATTEMPTS = 3 +_RABBITMQ_DEPTH_READ_INTERVAL_SECONDS = 0.05 + + +def _call_rabbitmq_with_retry( + rabbitmq_inspector: RabbitmqQueueInspector, *, stage: str, process_name: str, host: str, + port: int, credentials: EngineCredentials, queue: str, accept: Callable[[int], bool], + sleep: Callable[[float], None] = time.sleep, +) -> int: + """N15, p6-review-r2 (MINOR): the management API's `messages` field is fed by the node's own + stats collector (`collect_statistics_interval`, default 5000ms) — a depth read immediately + after a publish/reset can report a stale value in EITHER direction: 0 when a message really + is there (the direction that falsely PASSES an isolation check), or a leftover count when it + really is gone (the direction that falsely FAILS an `expected_depth` sentinel). `accept` is + the caller's own success predicate; retried a bounded few times before settling for whatever + the last read reported, so a genuinely wrong result is never masked by retrying forever — only + the collector's known lag window is absorbed. + """ + depth = 0 + for attempt in range(_RABBITMQ_DEPTH_READ_ATTEMPTS): + depth = _call_rabbitmq( + rabbitmq_inspector, stage=stage, process_name=process_name, host=host, port=port, + credentials=credentials, queue=queue, + ) + if accept(depth): + return depth + if attempt < _RABBITMQ_DEPTH_READ_ATTEMPTS - 1: + sleep(_RABBITMQ_DEPTH_READ_INTERVAL_SECONDS) + return depth + + +def _call_rabbitmq_action( + fn: Callable[..., None], *, stage: str, process_name: str, action: str, **kwargs: Any, +) -> None: + """Same B5 typing as `_call_rabbitmq`, for the write-side canary declare/publish call + `_run_canary_probe` makes through `context.rabbitmq_declare` — the m8 canary's OWN statement + against a store that has already passed readiness, exactly the class of call B5 covers. (Q10, + p6-review-r3: `context.rabbitmq_delete` is no longer one of these — N15 moved canary cleanup + to the reset-from-baseline path; the seam itself is kept, see `RabbitmqQueueDeleter`.)""" + try: + fn(**kwargs) + except Exception as exc: + raise ProcessRuntimeError( + stage, "store_statement_failed", + f"{process_name}: store rejected the provisioner's canary {action}: {exc}", + process=process_name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + + +def _rabbitmq_management_port(amqp_port: int) -> int: + """§2b's catalog/port formula fixes only the AMQP listener port; the management HTTP API + (needed to seed via `rabbitmqadmin` and to read queue depth for a sentinel) has no formula of + its own. A fixed `+10000` offset is a defensible, deterministic V1 default — `14000..14099` / + `15000..15799` (§2b) shifted by it lands at `24000..24099` / `25000..25799`, outside every + band this contract reserves, so it can never alias a different process's allocated port. Same + provisioner-internal status as `postgres_daemon_argv` et al.: swappable, never fixed by the + contract, never exercised against a real broker in this module's own test lane. + """ + return amqp_port + 10000 + + +def _rabbitmq_auth_header(credentials: EngineCredentials) -> str: + credential_bytes = f"{credentials.username}:{credentials.password}".encode("utf-8") + return "Basic " + base64.b64encode(credential_bytes).decode() + + +def default_rabbitmq_queue_inspector( + *, host: str, port: int, credentials: EngineCredentials, queue: str +) -> int: + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + request = urllib.request.Request(url) + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + try: + with urllib.request.urlopen(request, timeout=5.0) as response: + payload = json.loads(response.read()) + except urllib.error.HTTPError as exc: + if exc.code == 404: + # m8, p6-review-r1: a nonexistent queue is what "empty"/"absent" LOOKS like at this + # endpoint — the management API's own way of saying so is a 404, not an error the + # conformance gate's "never raises" promise should have to survive by accident. + return 0 + raise + return int(payload.get("messages", 0)) + + +def default_rabbitmq_queue_declare_and_publish( + *, host: str, port: int, credentials: EngineCredentials, queue: str, message: str, +) -> None: + """m8, p6-review-r1: the conformance canary previously only ever INSPECTED a rabbitmq queue — + since nothing ever created one, world 1's "is it visible" check compared against a queue that + never existed anywhere, a vacuous pass. Declares the reserved canary queue in world 0 for + real via the management API (never AMQP directly — same HTTP-only seam as every other + rabbitmq call this module makes) and publishes one message into it, so world 1's read is + checking something that could actually leak. + """ + management_port = _rabbitmq_management_port(port) + auth = _rabbitmq_auth_header(credentials) + + declare_url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + declare_request = urllib.request.Request( + declare_url, + data=json.dumps({"durable": False, "auto_delete": True}).encode("utf-8"), + method="PUT", + ) + declare_request.add_header("Content-Type", "application/json") + declare_request.add_header("Authorization", auth) + with urllib.request.urlopen(declare_request, timeout=5.0): + pass + + # The default exchange's routing key IS the queue name — the standard way the management + # API's own `publish` endpoint targets a specific queue without declaring a binding first. + publish_url = f"http://{host}:{management_port}/api/exchanges/%2F/amq.default/publish" + publish_request = urllib.request.Request( + publish_url, + data=json.dumps({ + "properties": {}, "routing_key": queue, "payload": message, + "payload_encoding": "string", + }).encode("utf-8"), + method="POST", + ) + publish_request.add_header("Content-Type", "application/json") + publish_request.add_header("Authorization", auth) + with urllib.request.urlopen(publish_request, timeout=5.0): + pass + + +def default_rabbitmq_queue_delete( + *, host: str, port: int, credentials: EngineCredentials, queue: str, +) -> None: + """Cleanup half of the canary (m8) — best-effort: a 404 means it is already gone (e.g. the + world-0 reset that runs right after already wiped it, since `datadir_copy` is rabbitmq's only + legal strategy), which is the desired end state, not a failure.""" + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/queues/%2F/{urlquote(queue, safe='')}" + request = urllib.request.Request(url, method="DELETE") + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + try: + with urllib.request.urlopen(request, timeout=5.0): + pass + except urllib.error.HTTPError as exc: + if exc.code != 404: + raise + + +class RabbitmqDefinitionsImporter(Protocol): + def __call__( + self, *, host: str, port: int, credentials: EngineCredentials, file: Path, + ) -> None: ... + + +def default_rabbitmq_definitions_importer( + *, host: str, port: int, credentials: EngineCredentials, file: Path, +) -> None: + """N17, p6-review-r2 (MINOR): `rabbitmqadmin import ` used to require the + `rabbitmqadmin` binary specifically — served BY the management plugin at runtime (fetched + from the running node's own web UI), never installed by the `rabbitmq-server` package itself, + and not in §0's guaranteed-binary list (`python`, `node`, `git`, `ffmpeg`, plus the §2b engine + catalog). `rabbitmqadmin import` is documented as a thin wrapper over exactly this endpoint + (`POST /api/definitions`) — the same management HTTP API this module already talks to for + every other rabbitmq call (`default_rabbitmq_queue_inspector` et al.), so seeding no longer + depends on a binary the snapshot might not ship at all. + """ + management_port = _rabbitmq_management_port(port) + url = f"http://{host}:{management_port}/api/definitions" + request = urllib.request.Request(url, data=file.read_bytes(), method="POST") + request.add_header("Content-Type", "application/json") + request.add_header("Authorization", _rabbitmq_auth_header(credentials)) + with urllib.request.urlopen(request, timeout=10.0): + pass + + +# --- §2c seed application ------------------------------------------------------------------- + + +def postgres_seed_argv(*, port: int, dbname: str, user: str, file: Path) -> list[str]: + return [ + "psql", "-h", "localhost", "-p", str(port), "-U", user, "-d", dbname, + "-v", "ON_ERROR_STOP=1", "-f", str(file), + ] + + +def postgres_seed_env(credentials: EngineCredentials) -> dict[str, str]: + return {"PGPASSWORD": credentials.password} + + +def redis_seed_argv(*, port: int) -> list[str]: + """No `-f`/script flag exists on `redis-cli`; a seed file is plain commands, one per line, + fed over stdin — the documented way `redis-cli` accepts a batch of ordinary commands (as + opposed to `--pipe`, which expects pre-encoded RESP, not this module's business to generate). + """ + return ["redis-cli", "-h", "localhost", "-p", str(port)] + + +def apply_seed_file( + engine: ManagedEngine, + file: Path, + *, + port: int, + dbname: str, + credentials: EngineCredentials | None, + process_name: str, + sync_run: Callable[..., subprocess.CompletedProcess], + user: int | None = None, + group: int | None = None, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, +) -> None: + """Applies one migration/seed file, per §2c: "applied in listed order." `postgres` shells out + to a psql-style command (`-f`, so a large schema file streams rather than loading into this + process); `redis`/`rabbitmq` use their own bulk-load mechanisms — three small per-engine + branches, mirroring `postgres_daemon_argv` et al.'s split, so a fourth catalog engine needs + only one new branch here, never a rewrite of the caller. `sync_run` is `SpawnContext.sync_run` + (or a test's fake) — the same "run one synchronous step, check its exit code" seam + `build_process_tree`'s build steps and `spawn_managed_process`'s postgres bootstrap already + use. Raises on the first failing file (`ProcessRuntimeError`, stage="seed"), never partially + proceeds silently past one — same rule `build_process_tree` holds for `build_commands`. + + B2, p6-review-r1: `user`/`group` are the store's OWN declared spawn identity (`svc-data`), + never left to default to the provisioner's own uid — `psql -f` honors backslash meta-commands + (`\\!`, `\\copy ... program`), so a migration/seed file is a customer-authored-content + execution path exactly like `build_commands`, which already drops privilege the same way. + """ + if engine is ManagedEngine.POSTGRES: + if credentials is None: + raise ProcessRuntimeError( + "seed", "internal_missing_credentials", + "postgres requires generated credentials to seed", process=process_name, + ) + argv = postgres_seed_argv(port=port, dbname=dbname, user=credentials.username, file=file) + # F12's own rule (§2b's closed env list), reapplied here: `env=` fully REPLACES a child's + # environment rather than extending the caller's — passing just `{"PGPASSWORD": ...}` + # would drop `PATH` entirely, and `psql` living anywhere outside `subprocess`'s POSIX + # fallback path (`/bin:/usr/bin`) — a homebrew or venv install, commonly — would silently + # fail to exec. + env = {**_allowlisted_ambient_env(os.environ), **postgres_seed_env(credentials)} + result = sync_run(argv, capture_output=True, text=True, env=env, user=user, group=group) + elif engine is ManagedEngine.REDIS: + argv = redis_seed_argv(port=port) + result = sync_run( + argv, capture_output=True, text=True, input=file.read_text(encoding="utf-8"), + user=user, group=group, + ) + elif engine is ManagedEngine.RABBITMQ: + if credentials is None: + raise ProcessRuntimeError( + "seed", "internal_missing_credentials", + "rabbitmq requires generated credentials to seed", process=process_name, + ) + # N17, p6-review-r2: seeded over the management HTTP API directly, never through a + # subprocess/`sync_run` — no `rabbitmqadmin` binary dependency (`user`/`group` are moot + # here for the same reason: an HTTP call has no OS-level identity to drop). + try: + rabbitmq_import(host="localhost", port=port, credentials=credentials, file=file) + except Exception as exc: + raise ProcessRuntimeError( + "seed", "seed_failed", f"{file}: {exc}", process=process_name, + domain=FailureDomain.ENVIRONMENT, + ) from exc + return + else: # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + raise ProcessRuntimeError( + "seed", "seed_failed", f"unknown engine {engine!r}", process=process_name, + domain=FailureDomain.ENVIRONMENT, + ) + if result.returncode != 0: + stderr = (result.stderr or "").strip()[:2000] + raise ProcessRuntimeError( + "seed", "seed_failed", + f"{file}: exited {result.returncode}" + (f": {stderr}" if stderr else ""), + process=process_name, domain=FailureDomain.ENVIRONMENT, + ) + + +def apply_store_seed( + store: StoreEntry, + *, + engine: ManagedEngine, + bundle_dir: Path, + port: int, + dbname: str, + credentials: EngineCredentials | None, + process_name: str, + sync_run: Callable[..., subprocess.CompletedProcess], + user: int | None = None, + group: int | None = None, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, +) -> None: + """§2c: "migrations then seed_files... applied in listed order" — migrations always precede + seed_files, regardless of how many files either list holds, and each list keeps its own + authored order (never sorted).""" + for relative_path in (*store.migrations, *store.seed_files): + apply_seed_file( + engine, bundle_dir / relative_path, port=port, dbname=dbname, credentials=credentials, + process_name=process_name, sync_run=sync_run, user=user, group=group, + rabbitmq_import=rabbitmq_import, + ) + + +# --- §2c sentinel checking --------------------------------------------------------------------- + + +def check_sentinel( + store: StoreEntry, + *, + engine: ManagedEngine, + host: str, + port: int, + dbname: str | None, + credentials: EngineCredentials | None, + sql_runner: SqlRunner, + redis_runner: RedisCommandRunner, + rabbitmq_inspector: RabbitmqQueueInspector, + process_name: str = "", + stage: str = "sentinel", +) -> bool: + """§2c/§4.2: "a read-only check plus its exact expected value (string compare)." Dispatches on + the sentinel's own implied shape (`Sentinel.implied_engine`), which the model layer already + guarantees matches the store's engine (`sentinel_shape_mismatch`) — this only has to act on + it, not re-verify it. `process_name`/`stage` are for B5's typed-failure wrapper only (default + to the empty string / "sentinel" so every pre-existing direct caller keeps working unchanged). + """ + sentinel = store.sentinel + if engine is ManagedEngine.POSTGRES: + if credentials is None or dbname is None: + return False + rows = _call_sql( + sql_runner, stage=stage, process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement=sentinel.query, # type: ignore[arg-type] + read_only=True, # N8, p6-review-r2: customer-authored content, §2c calls this a read. + ) + actual = str(rows[0][0]) if rows and rows[0] else None + return actual == sentinel.expected + if engine is ManagedEngine.REDIS: + value = _call_redis( + redis_runner, stage=stage, process_name=process_name, + host=host, port=port, command=["GET", sentinel.key], # type: ignore[list-item] + ) + actual = value.decode() if isinstance(value, bytes) else (None if value is None else str(value)) + return actual == sentinel.expected + if engine is ManagedEngine.RABBITMQ: + if credentials is None: + return False + depth = _call_rabbitmq_with_retry( + rabbitmq_inspector, stage=stage, process_name=process_name, + host=host, port=port, credentials=credentials, queue=sentinel.queue, # type: ignore[arg-type] + accept=lambda value: value == sentinel.expected_depth, # N15, p6-review-r2. + ) + return depth == sentinel.expected_depth + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + +# --- per-world spawn orchestration -------------------------------------------------------------- + + +@dataclass(frozen=True) +class SpawnContext: + """The caller-supplied dependencies `spawn_world` needs, grouped so the per-call signature + stays to (manifest, world_index, shared_handles) — Phase 6's `provision()` builds one of + these per job and calls `spawn_world` once per world index inside its own reconciliation + loop; that loop, idempotency across retries, and `EnvironmentRuntime` state assembly across + all `instances` worlds are its job, not this function's. + """ + + work_directory: Path + port_plan: PortPlan + credentials: dict[str, EngineCredentials] + secret_values: dict[str, str] + secret_purposes: dict[str, str] + runner: ProcessRunner = default_process_runner + # Shared by `build_process_trees`' `build_commands` steps and `spawn_managed_process`'s + # postgres `initdb` bootstrap — both are "run one synchronous step, check its exit code." + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run + prober: CapabilityProber = default_capability_prober + copy: Callable[[Path, Path], None] | None = None + # F1, p5-round1-review: threaded to every build/spawn call this context drives. + # `require_declared_user=False` is the local-lane default (no `svc-*` accounts on a dev box); + # a hosted caller sets it `True` so a snapshot that somehow lacks a declared user fails typed + # instead of silently running unprivileged. + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver + require_declared_user: bool = False + chown: Callable[[Path, int, int], None] = _default_chown + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS + # §2c/§5 store command seams (below) — defaulted so every existing `SpawnContext(...)` call + # in P5's own tests keeps working unchanged; Phase 6 callers override them with spies. + sql_runner: SqlRunner = default_sql_runner + redis_runner: RedisCommandRunner = default_redis_command_runner + rabbitmq_inspector: RabbitmqQueueInspector = default_rabbitmq_queue_inspector + # m8, p6-review-r1: the conformance canary's rabbitmq branch (below `_run_canary_probe`) + # needs to actually create the reserved queue, not just read one that was never declared. + rabbitmq_declare: RabbitmqQueueDeclarer = default_rabbitmq_queue_declare_and_publish + rabbitmq_delete: RabbitmqQueueDeleter = default_rabbitmq_queue_delete + # N17, p6-review-r2: seeding no longer shells out to `rabbitmqadmin` — the same management + # HTTP API seam every other rabbitmq call in this context already uses. + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer + bundle_dir: Path | None = None + + +@dataclass +class WorldSpawnResult: + handles: dict[str, SpawnedWorldProcess] + endpoints: dict[str, RuntimeEndpoint] + + +def spawn_world( + manifest: EnvironmentBundleV2, + *, + world_index: int, + context: SpawnContext, + shared_handles: dict[str, SpawnedWorldProcess] | None = None, +) -> WorldSpawnResult: + """Spawns every process for ONE world, in `depends_on` dependency order, waiting on each + dependency's readiness before starting its dependent (§2b). `shared_handles` carries + already-running job-shared managed engines (from an earlier world's call) so they are reused, + never respawned, for `world_index > 0` — the caller is expected to thread the same dict + through every `spawn_world` call for one job. Source process build trees must already exist + (via `build_process_tree`, called once per process before any world is spawned) — this + function only creates per-world scratch directories, never a build tree. + """ + endpoints = build_endpoints( + manifest, + world_index=world_index, + port_plan=context.port_plan, + credentials=context.credentials, + ) + configuration_addresses = configuration_addresses_from_endpoints(endpoints) + processes_by_name = {process.name: process for process in manifest.processes} + handles: dict[str, SpawnedWorldProcess] = dict(shared_handles or {}) + + try: + for name in _topological_order(manifest): + if name in handles: + continue # a job-shared managed engine already running from an earlier world. + process = processes_by_name[name] + for dependency_name in process.depends_on: + wait_for_dependency( + manifest, + dependency_name, + world_index=world_index, + port_plan=context.port_plan, + spawned=handles[dependency_name], + credentials=context.credentials, + prober=context.prober, + ) + if isinstance(process, ManagedProcess): + data_dir = managed_engine_data_dir( + context.work_directory, + name, + world_index=None if context.port_plan.is_job_shared(name) else world_index, + ) + handle = spawn_managed_process( + process, + port=context.port_plan.port_for(name, world_index), + data_dir=data_dir, + credentials=context.credentials.get(name), + runner=context.runner, + sync_run=context.sync_run, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + else: + handle = spawn_source_process( + process, + build_dir=build_tree_dir(context.work_directory, name), + world_dir=world_scratch_dir(context.work_directory, world_index, name), + world_index=world_index, + port_plan=context.port_plan, + configuration_addresses=configuration_addresses, + secret_values=context.secret_values, + secret_purposes=context.secret_purposes, + runner=context.runner, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + ) + handles[name] = handle + except BaseException as exc: + # N4, p6-review-r2 (MAJOR): a `depends_on_timeout`/`spawn_failed` partway through this + # world's own process list used to drop every ALREADY-spawned process of the SAME world on + # the floor — nothing held it, so a caller's later `close()`/retry could `rmtree` a data + # directory, or attempt a fresh spawn, out from under a still-live sibling process. + # `handles` (this world's own accumulated dict, seeded from `shared_handles`) is attached + # to the exception so the caller can publish/terminate it instead. + exc.partial_handles = handles # type: ignore[attr-defined] + raise + return WorldSpawnResult(handles=handles, endpoints=endpoints) + + +def build_process_trees( + manifest: EnvironmentBundleV2, *, source_root: Path, context: SpawnContext +) -> dict[str, Path]: + """Builds every `source` process's tree once — the caller invokes this exactly once per job, + before the first `spawn_world` call for any world (§2b: `build_commands` run "once per + job").""" + return { + process.name: build_process_tree( + process, + source_root=source_root, + build_root=context.work_directory / "build", + run=context.sync_run, + copy=context.copy, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, + chown=context.chown, + build_step_timeout_seconds=context.build_step_timeout_seconds, + ) + for process in manifest.processes + if isinstance(process, SourceProcess) + } + + +# --- §4.3 healthy() -------------------------------------------------------------------------- + + +def _split_host_port(address: str) -> tuple[str, int]: + parsed = urlsplit(address) + return parsed.hostname or "localhost", parsed.port or 0 + + +def _postgres_credentials_from_address(address: str) -> tuple[str | None, str | None, str | None]: + """F9: the rendered `endpoint.address` for a postgres capability already carries the + generated role/password/database (`postgresql://harness:@localhost:/w`) — parsed + back out rather than threading `EngineCredentials` separately into `probe_runtime_health`, + which only ever sees `EnvironmentRuntime`, not the job's credential map.""" + parsed = urlsplit(address) + return parsed.username, parsed.password, (parsed.path.lstrip("/") or None) + + +def probe_runtime_health( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber = default_capability_prober, +) -> bool: + """§4 point 3: "`healthy` = declared `readiness` probes, not 'process is running.'" Every + declared `readiness` entry for a capability this runtime actually exposes must pass; a + capability with no readiness entry declared carries no health obligation (§2b: "Health/ + readiness is otherwise declared ONLY in the capability-level `readiness` section").""" + for probe in manifest.readiness: + endpoint = runtime.endpoints.get(probe.capability) + capability = manifest.capabilities.get(probe.capability) + if endpoint is None or capability is None: + return False + host, port = _split_host_port(endpoint.address) + user = password = dbname = None + if capability.protocol is CapabilityProtocol.POSTGRES: + user, password, dbname = _postgres_credentials_from_address(endpoint.address) + if not prober( + protocol=capability.protocol, host=host, port=port, path=probe.path, + user=user, password=password, dbname=dbname, + ): + return False + return True + + +async def healthy( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber = default_capability_prober, +) -> bool: + """Async wrapper matching §4's `RuntimeProvider.healthy` shape (`runtime.py`'s own + `LocalComposeRuntimeProvider.healthy` uses the same `asyncio.to_thread` idiom). + + §3's `state` transitions are fixed: `preparing->ready`, `ready->unhealthy`, and + `unhealthy->ready` ONLY via re-provision reconcile (§4 point 1: "a sick world mid-job is + recovered by calling `provision` again"). `healthy()` is not a reconcile (F6, p5-round1- + review) — it may only ever DEMOTE `runtime.state`, never promote it. `ready` stays `ready` + while still healthy; `unhealthy`/`stopped` are cleared only by `provision()`/`close()`, never + by a passing probe alone (a world whose sentinel was never re-proved after coming back up must + not silently re-enter the pool). + """ + import asyncio + + is_healthy = await asyncio.to_thread(probe_runtime_health, manifest, runtime, prober=prober) + if not is_healthy: + runtime.state = RuntimeState.UNHEALTHY + elif runtime.state is RuntimeState.PREPARING: + runtime.state = RuntimeState.READY + return is_healthy + + +# --- internal invariant codes ------------------------------------------------------------------- +# +# Neither is in §2f's closed table — both mark a precondition upstream layers already guarantee +# (a postgres/rabbitmq `ManagedProcess` always gets generated credentials; a store's backing +# service is always a `ManagedProcess`, per `bundle_v2`'s own `store_service_not_managed`), same +# status as `render_capability_address`'s `internal_missing_credentials` above — a bug to fix +# here, never a bundle defect the outbound seam needs a name for. +_INTERNAL_MISSING_CREDENTIALS = "internal_missing_credentials" +_INTERNAL_INVARIANT_VIOLATED = "internal_invariant_violated" + + +def _require_credentials( + credentials: EngineCredentials | None, *, stage: str, process_name: str +) -> EngineCredentials: + if credentials is None: + raise ProcessRuntimeError( + stage, _INTERNAL_MISSING_CREDENTIALS, + f"{process_name}: postgres/rabbitmq requires generated credentials but none were " + "supplied", process=process_name, + ) + return credentials + + +# --- §2c baseline: store lookup + naming ----------------------------------------------------- + + +def _store_by_process_name(manifest: EnvironmentBundleV2) -> dict[str, StoreEntry]: + """Every declared store, keyed by the `ManagedProcess.name` backing its capability — the + shape every baseline/clone/reset function below actually wants, one hop past what §2c's own + `seed.stores[].capability` names.""" + result: dict[str, StoreEntry] = {} + if manifest.seed is not None: + for store in manifest.seed.stores: + capability = manifest.capabilities[store.capability] + result[capability.service] = store + return result + + +def _template_database_name(process_name: str) -> str: + """§2c's own baseline databases are never `w` (§2b reserves that shape for per-world logical + DBs) — `-` is replaced since a process `name` may carry it (`^[a-z0-9][a-z0-9_-]*$`, §2b) but a + bare postgres identifier cannot without quoting.""" + return f"alk_baseline_{process_name.replace('-', '_')}" + + +def _datadir_copy_baseline_path(work_directory: Path, process_name: str) -> Path: + """A sibling of the live `/work/managed//` dir (§0), never nested inside it — `freeze_ + baseline` copies FROM the live dir INTO this one; nesting would make that copy recurse into + its own destination.""" + return _ensure_within( + work_directory / "managed" / f"{process_name}.baseline", work_directory, + process_name=process_name, stage="baseline", + ) + + +def _measure_postgres_row_counts( + *, host: str, port: int, credentials: EngineCredentials, dbname: str, sql_runner: SqlRunner, + process_name: str, +) -> dict[str, int]: + """Per-table row counts as the baseline stands right after seeding — `HostedWorld`'s own + `baseline_row_counts` input (`world/handle.py`'s `STATE_ROW_CAP` gate), measured once here + rather than re-measured at scenario time, so the cap it enforces cannot depend on what a + scenario already wrote to the table.""" + tables = _call_sql( + sql_runner, stage="baseline", process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement="SELECT tablename FROM pg_tables WHERE schemaname = 'public'", + read_only=True, # N8, p6-review-r2. + ) + counts: dict[str, int] = {} + for row in tables: + table = row[0] + # M1, p6-review-r1: `table` is the raw `pg_tables.tablename` — from the untrusted repo's + # own migrations. An unescaped double quote inside it used to close the identifier early + # and let anything after it execute as a second statement (psycopg3's simple-query + # protocol runs every statement in an unparameterized `execute()` call). + safe_table = table.replace('"', '""') + result = _call_sql( + sql_runner, stage="baseline", process_name=process_name, + host=host, port=port, user=credentials.username, password=credentials.password, + dbname=dbname, statement=f'SELECT COUNT(*) FROM "{safe_table}"', + read_only=True, # N8, p6-review-r2. + ) + counts[table] = int(result[0][0]) if result and result[0] else 0 + return counts + + +# --- B4, p6-review-r1: readiness wait between a managed-engine spawn and the first statement -- + +_DEFAULT_STORE_READY_TIMEOUT_SECONDS = 30.0 +_DEFAULT_STORE_READY_INTERVAL_SECONDS = 0.25 + +_PROTOCOL_BY_ENGINE = { + ManagedEngine.POSTGRES: CapabilityProtocol.POSTGRES, + ManagedEngine.REDIS: CapabilityProtocol.REDIS, + ManagedEngine.RABBITMQ: CapabilityProtocol.AMQP, +} + + +def _wait_for_store_ready( + manifest: EnvironmentBundleV2, + process: ManagedProcess, + *, + port: int, + context: SpawnContext, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> None: + """B4, p6-review-r1: `wait_for_dependency` already makes every `depends_on` EDGE wait for its + dependency's readiness before touching it; a freeze/seal call site is not itself anyone's + declared dependent, so the very same class of race (the engine has just been `exec`'d and has + not finished booting) went unguarded at every place THIS module spawns a store immediately + before issuing its own first statement against it. Reuses the manifest's declared `readiness` + timeout for this process's capability when the bundle names one, else a bounded default — + `default_capability_prober` already degrades to a bare TCP probe without credentials, so this + never requires a real login to work. Raises `depends_on_timeout` (§2f) on exhaustion, the same + code `wait_for_dependency` uses for the identical class of failure. + """ + probes = _readiness_probes_for_process(manifest, process.name) + timeout = max( + (probe.timeout_seconds for probe in probes), default=_DEFAULT_STORE_READY_TIMEOUT_SECONDS + ) + interval = min( + (probe.interval_seconds for probe in probes), default=_DEFAULT_STORE_READY_INTERVAL_SECONDS + ) + credentials = context.credentials.get(process.name) + user = credentials.username if credentials else None + password = credentials.password if credentials else None + # Always the default `postgres` database — guaranteed to exist the instant `initdb` finishes, + # regardless of which baseline/world database this particular spawn is ultimately building + # toward. This call only has to prove "the engine accepts connections." + dbname = "postgres" if process.engine is ManagedEngine.POSTGRES else None + protocol = _PROTOCOL_BY_ENGINE[process.engine] + + def ready() -> bool: + if not context.prober( + protocol=protocol, host="localhost", port=port, path=None, + user=user, password=password, dbname=dbname, + ): + return False + if process.engine is ManagedEngine.RABBITMQ: + # N7, p6-review-r2: the AMQP listener probed above comes up during CORE boot; the + # management plugin's HTTP listener comes up in the PLUGIN boot step that follows — + # the same B4 race, unfixed for the one engine whose seed/sentinel/canary statements + # all travel over the listener that was never probed. Both must answer. + if not context.prober( + protocol=CapabilityProtocol.HTTP, host="localhost", + port=_rabbitmq_management_port(port), path="/api/overview", + user=user, password=password, + ): + return False + return True + + _poll_until( + ready, timeout=timeout, interval=interval, clock=clock, sleep=sleep, + timeout_error=lambda: ProcessRuntimeError( + "baseline", "depends_on_timeout", + f"{process.name}: did not become ready within {timeout}s", process=process.name, + domain=FailureDomain.INFRASTRUCTURE, + ), + ) + + +# --- N2, p6-review-r2 (BLOCKER): poll the promote-path health check, don't sample it once ------- + + +def _poll_runtime_health( + manifest: EnvironmentBundleV2, + runtime: EnvironmentRuntime, + *, + prober: CapabilityProber, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, +) -> bool: + """Shared by BOTH promote sites — `provision()`'s PREPARING->READY promotion and `reset()`'s + post-sentinel probe — which used to call `probe_runtime_health` exactly once, immediately, + with no wait. `spawn_world` only waits on `depends_on` EDGES (round-1's own M4 note): nobody + depends on the DAG's TERMINAL process, so nothing ever waited for it. A bundle whose + readiness-bearing process is terminal (§2a's own documented `vapi`/`retell`-with-a-backend + shape) was marked `UNHEALTHY` on every provision and every reset regardless of how quickly it + actually came up, driving §5.4's `world_pool_exhausted` against a healthy environment. + + Same `max(declared timeout, default)` / `min(declared interval, default)` shape `_wait_for_ + store_ready` already established for the identical class of race, applied across every probe + the runtime's OWN endpoints declare (`manifest.readiness`, the same set `probe_runtime_health` + itself iterates). Exhaustion returns `False` — never raises: this feeds a state DECISION + (`READY` vs `UNHEALTHY`), not a provisioning failure the caller must escalate. + """ + timeout = max( + (probe.timeout_seconds for probe in manifest.readiness), + default=_DEFAULT_STORE_READY_TIMEOUT_SECONDS, + ) + interval = min( + (probe.interval_seconds for probe in manifest.readiness), + default=_DEFAULT_STORE_READY_INTERVAL_SECONDS, + ) + deadline = clock() + timeout + while True: + if probe_runtime_health(manifest, runtime, prober=prober): + return True + if clock() >= deadline: + return False + sleep(interval) + + +# --- §5.3 build output (`build.json`) --------------------------------------------------------- + + +@dataclass(frozen=True) +class StoreBaselineRecord: + """One store's sealed identity, as §5.3 wants it recorded on the build output. + `baseline_reference`: `template_database` -> the sealed template database's own name; + `datadir_copy` -> the baseline snapshot directory world-clone/reset copy from; + `empty` -> `""` (§5.3: "no-op capture" — there is nothing to seal). `row_counts` is populated + for postgres stores only — `HostedWorld` (the only consumer) is postgres-only.""" + + capability: str + process_name: str + engine: ManagedEngine + strategy: BaselineStrategy + inputs_digest: str + baseline_reference: str + row_counts: dict[str, int] + + def to_json(self) -> dict[str, JsonValue]: + return { + "capability": self.capability, + "process_name": self.process_name, + "engine": self.engine.value, + "strategy": self.strategy.value, + "inputs_digest": self.inputs_digest, + "baseline_reference": self.baseline_reference, + "row_counts": dict(self.row_counts), + } + + +@dataclass +class BuildOutput: + """§5.3's `build.json` artifact: "record `inputs_digest` + achieved-baseline reference on the + build output." `conformance`/`conformance_reason` start unset; the conformance gate fills them + in and the caller re-writes the file — §4's own "record pass/fail on the build output." + `requested_parallelism`/`effective_parallelism`/`degrade_reason` (m1, p6-review-r1) are what + P7's own `parallelism_degraded` event payload needs (`{requested, effective, reason}`) for + EITHER degrade cause — `PortPlan.degraded_reason` (`fixed_port`) used to be computed and never + read by this provider; only the conformance gate's own reason ever reached `build.json`. + """ + + bundle_digest: str + stores: list[StoreBaselineRecord] + conformance: bool | None = None + conformance_reason: str | None = None + requested_parallelism: int | None = None + effective_parallelism: int | None = None + degrade_reason: str | None = None + + def to_json(self) -> dict[str, JsonValue]: + return { + "bundle_digest": self.bundle_digest, + "stores": [record.to_json() for record in self.stores], + "conformance": self.conformance, + "conformance_reason": self.conformance_reason, + "requested_parallelism": self.requested_parallelism, + "effective_parallelism": self.effective_parallelism, + "degrade_reason": self.degrade_reason, + } + + +def write_build_output(work_directory: Path, build_output: BuildOutput) -> Path: + artifacts_dir = work_directory / "artifacts" + artifacts_dir.mkdir(parents=True, exist_ok=True) + target = artifacts_dir / "build.json" + target.write_text( + json.dumps(build_output.to_json(), indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return target + + +# --- §5 step 3: baseline freeze ----------------------------------------------------------------- + + +@dataclass +class FreezeResult: + """`freeze_baseline`'s full output: the `BuildOutput` §5.3 wants written to `build.json`, plus + the live handles of every `template_database` engine it just sealed — that engine stays + running for the rest of the job (§2b: job-shared), so the caller (`ProcessRuntimeProvider`) + must carry its handle forward into every world rather than re-spawning it.""" + + build_output: BuildOutput + job_shared_handles: dict[str, SpawnedWorldProcess] + + +def freeze_baseline( + manifest: EnvironmentBundleV2, *, bundle_digest: str, context: SpawnContext, +) -> FreezeResult: + """§5 step 3 / §2c: seed each managed store once, then seal it per baseline strategy — the + FIRST seal; every later world clone/reset reuses what this produces (`_seal_world_store`). + Runs once per job, before any world exists — the caller is responsible for calling this + exactly once, same convention as `build_process_trees`. `context.bundle_dir` must be set (the + bundle's own directory, for locating `migrations`/`seed_files` on disk) — the one place in + this module a caller must supply it, since seeding is the one place they matter. + """ + if context.bundle_dir is None: + raise ProcessRuntimeError( + "baseline", _INTERNAL_INVARIANT_VIOLATED, "SpawnContext.bundle_dir is required to seed", + ) + store_by_process = _store_by_process_name(manifest) + records: list[StoreBaselineRecord] = [] + job_shared_handles: dict[str, SpawnedWorldProcess] = {} + try: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + store = store_by_process.get(process.name) + if store is None: + continue # §2b default: no store entry -> per-world, nothing to freeze here. + record, handle = _freeze_one_store(manifest, process, store, context=context) + records.append(record) + if handle is not None: + job_shared_handles[process.name] = handle + except BaseException: + # N4, p6-review-r2 (MAJOR): a raise partway through leaves every EARLIER store's own + # job-shared engine live and unreferenced anywhere — `self._job_shared_handles` is never + # assigned on a raise (only after this function RETURNS), so nothing can terminate it and + # nothing frees the port it holds for a same-bundle retry. Terminated here rather than + # left running: the whole attempt has already failed (`ProcessRuntimeProvider` resets the + # job identity on this same raise — N3), so nothing downstream can resume these handles. + for handle in job_shared_handles.values(): + _terminate_and_wait(handle.handle) + raise + return FreezeResult( + build_output=BuildOutput(bundle_digest=bundle_digest, stores=records), + job_shared_handles=job_shared_handles, + ) + + +def _freeze_one_store( + manifest: EnvironmentBundleV2, process: ManagedProcess, store: StoreEntry, *, + context: SpawnContext, +) -> tuple[StoreBaselineRecord, SpawnedWorldProcess | None]: + strategy = store.baseline.strategy + if strategy is BaselineStrategy.EMPTY: + # §5.3: "no-op capture" — nothing seeded, nothing sealed here; `_seal_world_store`'s + # fallback branch (re)establishes this store's declared state on every clone/reset instead. + return ( + StoreBaselineRecord( + capability=store.capability, process_name=process.name, engine=process.engine, + strategy=strategy, inputs_digest=store.baseline.inputs_digest, + baseline_reference="", row_counts={}, + ), + None, + ) + + port = context.port_plan.port_for(process.name, 0) # job-shared or not, world 0's slot is + # this bootstrap instance's own — a job-shared engine's port never varies by world_index + # (`PortPlan.port_for`); a `datadir_copy` engine has no real world yet to collide with. + credentials = context.credentials.get(process.name) + data_dir = managed_engine_data_dir(context.work_directory, process.name, world_index=None) + handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + try: + # B4, p6-review-r1: the engine has just been `exec`'d — the very next thing this function + # does is talk to it. + _wait_for_store_ready(manifest, process, port=port, context=context) + + baseline_dbname = ( + _template_database_name(process.name) if process.engine is ManagedEngine.POSTGRES + else "" + ) + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials( + credentials, stage="baseline", process_name=process.name + ) + _call_sql( + context.sql_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, + dbname="postgres", statement=f'CREATE DATABASE "{baseline_dbname}"', + ) + apply_store_seed( + store, engine=process.engine, bundle_dir=context.bundle_dir, port=port, + dbname=baseline_dbname, credentials=credentials, process_name=process.name, + sync_run=context.sync_run, user=handle.uid, group=handle.gid, + rabbitmq_import=context.rabbitmq_import, + ) + # m3, p6-review-r1: §2c defines the sentinel as a check "against the freshly seeded + # baseline" — checked here, before sealing, so a seed that silently produced the wrong + # state is caught at freeze (surfaced as `seed_failed` — the seed's own content did not + # produce what its own sentinel expects, a deterministic authoring fault) rather than + # first noticed at the scheduler's first `reset`, long after `build.json` already + # recorded success. + sentinel_dbname = baseline_dbname if process.engine is ManagedEngine.POSTGRES else None + if not check_sentinel( + store, engine=process.engine, host="localhost", port=port, dbname=sentinel_dbname, + credentials=credentials, sql_runner=context.sql_runner, + redis_runner=context.redis_runner, + rabbitmq_inspector=context.rabbitmq_inspector, process_name=process.name, + stage="baseline", + ): + raise ProcessRuntimeError( + "baseline", "seed_failed", + f"{process.name}: sentinel check failed against the freshly seeded baseline", + process=process.name, domain=FailureDomain.ENVIRONMENT, + ) + + row_counts: dict[str, int] = {} + baseline_reference = "" + result_handle: SpawnedWorldProcess | None = None + if strategy is BaselineStrategy.TEMPLATE_DATABASE: + if process.engine is ManagedEngine.POSTGRES: + row_counts = _measure_postgres_row_counts( + host="localhost", port=port, credentials=credentials, dbname=baseline_dbname, + sql_runner=context.sql_runner, process_name=process.name, + ) + # Sealed: marked a TEMPLATE and closed to new connections, so nothing can write into + # it after this — "the sealed post-migrate+seed datastore state" (glossary, "Baseline"). + _call_sql( + context.sql_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, + dbname="postgres", + statement=( + f'ALTER DATABASE "{baseline_dbname}" WITH IS_TEMPLATE true ' + "ALLOW_CONNECTIONS false" + ), + ) + baseline_reference = baseline_dbname + result_handle = handle # stays running — job-shared for the rest of the job. + elif strategy is BaselineStrategy.DATADIR_COPY: + if process.engine is ManagedEngine.POSTGRES: + row_counts = _measure_postgres_row_counts( + host="localhost", port=port, credentials=credentials, dbname=baseline_dbname, + sql_runner=context.sql_runner, process_name=process.name, + ) + if process.engine is ManagedEngine.REDIS: + # Q1, p6-review-r3 (MAJOR): `redis_daemon_argv` disables save points (`--save + # ""`), so a plain SIGTERM shutdown persists nothing — without an explicit + # synchronous `SAVE` first, the copied data dir below is an empty baseline. + _call_redis( + context.redis_runner, stage="baseline", process_name=process.name, + host="localhost", port=port, command=["SAVE"], + ) + # M7, p6-review-r1: waits for the engine to actually exit before the snapshot copy + # below — a bare `terminate()` only sends SIGTERM, and postgres's smart shutdown may + # not have even started, let alone finished, by the next line. Q12, p6-review-r3: a + # longer wait for rabbitmq — its broker shutdown routinely exceeds the 5s default and + # a SIGKILL mid-mnesia-write would seal a corrupt snapshot. + _terminate_and_wait( + handle.handle, + timeout=( + _RABBITMQ_TERMINATE_WAIT_SECONDS if process.engine is ManagedEngine.RABBITMQ + else _TERMINATE_WAIT_SECONDS + ), + ) + baseline_dir = _datadir_copy_baseline_path(context.work_directory, process.name) + try: + if baseline_dir.exists(): + shutil.rmtree(baseline_dir) + (context.copy or _copytree_preserving_symlinks)(data_dir, baseline_dir) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): §4.6 — this IS the "baseline/seal copies" half of the + # mapping; sealing the very first snapshot every world clones/resets from is a + # filesystem operation, not a store-command seam B5 already typed. + raise ProcessRuntimeError( + "baseline", "store_statement_failed", + f"{process.name}: sealing the datadir_copy baseline: {exc}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + baseline_reference = str(baseline_dir) + # `result_handle` stays `None` — every world starts its OWN engine instance from this + # snapshot (`_seal_world_store`'s `DATADIR_COPY` branch), never this bootstrap one. + + record = StoreBaselineRecord( + capability=store.capability, process_name=process.name, engine=process.engine, + strategy=strategy, inputs_digest=store.baseline.inputs_digest, + baseline_reference=baseline_reference, row_counts=row_counts, + ) + return record, result_handle + except BaseException: + # N4, p6-review-r2 (MAJOR): any raise between spawn and the successful return (a readiness + # timeout, a bad seed file, a failing freeze-time sentinel) used to leave THIS store's own + # just-spawned engine live and referenced NOWHERE — `freeze_baseline`'s own loop never + # even sees `handle` when this function raises instead of returning. Best-effort: + # `DATADIR_COPY`'s own success path has usually already terminated it by this point, and + # `_terminate_and_wait` tolerates an already-dead handle. + _terminate_and_wait(handle.handle) + raise + + +# --- §4.2/§4 world clone + reset ----------------------------------------------------------------- + + +def _seal_world_store( + manifest: EnvironmentBundleV2, + process: ManagedProcess, + store: StoreEntry | None, + record: StoreBaselineRecord | None, + *, + world_index: int, + context: SpawnContext, +) -> SpawnedWorldProcess | None: + """Drives ONE world's managed store to the sealed baseline, per §4.2 — shared by the FIRST + clone (`ProcessRuntimeProvider._ensure_world`) and every later `reset_world`; the caller is + expected to have already terminated any existing per-world handle for `process.name` before + calling this again. Returns the new per-world `SpawnedWorldProcess` for a per-world engine + (`datadir_copy`/`empty`/no-entry), or `None` for a job-shared `template_database` engine — + nothing new to spawn there, only its logical `wN` database changes. + """ + port = context.port_plan.port_for(process.name, world_index) + data_dir = managed_engine_data_dir(context.work_directory, process.name, world_index=world_index) + credentials = context.credentials.get(process.name) + strategy = store.baseline.strategy if store is not None else None + + if strategy is BaselineStrategy.TEMPLATE_DATABASE: + if record is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process.name}: template_database has no frozen baseline record", + process=process.name, + ) + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + # No spawn on this branch (the job-shared engine is already running, and already went + # through B4's wait once at freeze/first-clone time) — nothing new to wait on here. + _reset_template_database( + host="localhost", port=port, credentials=credentials, world_db=f"w{world_index}", + template_db=record.baseline_reference, sql_runner=context.sql_runner, + process_name=process.name, + ) + return None + + if strategy is BaselineStrategy.DATADIR_COPY: + # R1, p6-review-r4 (known defect, disclosed rather than redesigned this phase): for + # rabbitmq specifically, this branch restores the mnesia DIRECTORY but not a schema + # rabbitmq will actually adopt — mnesia data is node-name-bound and per-world node names + # must stay distinct for epmd (see `rabbitmq_daemon_env`'s docstring), so a rabbitmq + # `datadir_copy` baseline boots correctly in world 0 only. Recorded follow-up: a + # definitions-export restore via `default_rabbitmq_definitions_importer`. + if record is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process.name}: datadir_copy has no frozen baseline record", process=process.name, + ) + try: + if data_dir.exists(): + shutil.rmtree(data_dir) + # No pre-`mkdir` here (unlike the fallback branch below) — `copytree`-shaped `copy` + # creates its own destination and raises `FileExistsError` if it is already there. + (context.copy or _copytree_preserving_symlinks)( + Path(record.baseline_reference), data_dir + ) + except (OSError, shutil.Error) as exc: + # N9, p6-review-r2 (MAJOR): restoring a world's engine from its sealed baseline + # snapshot is itself an infrastructure operation (§4.6) — the "baseline/seal copies" + # half of the mapping, same class `_freeze_one_store`'s own snapshot copy got. + raise ProcessRuntimeError( + "reset", "store_statement_failed", + f"{process.name}: restoring the datadir_copy baseline: {exc}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + new_handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + # B4, p6-review-r1: fresh spawn from a just-restored data directory — the next line talks + # to it immediately. + _wait_for_store_ready(manifest, process, port=port, context=context) + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + world_db = f"w{world_index}" + baseline_dbname = _template_database_name(process.name) + # §2b: "under `datadir_copy` the provisioner configures each per-world engine's + # database to `w` as well — one rule, both strategies." The copied dir's own + # database still carries `_freeze_one_store`'s bootstrap name; renamed here, once per + # world, never touching the pristine snapshot every OTHER world/reset still copies from. + _call_sql( + context.sql_runner, stage="reset", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'ALTER DATABASE "{baseline_dbname}" RENAME TO "{world_db}"', + ) + return new_handle + + # `empty` strategy, or no `seed.stores` entry at all: no baseline snapshot exists to copy back + # in (`freeze_baseline` never ran for it — §5.3's "no-op capture" / §2b's per-world default), + # so this always starts the engine over a WIPED, empty data directory — unconditionally, since + # a bare restart alone cannot be trusted to flush every engine (rabbitmq persists to disk + # regardless of redis's own `--save ""`). An explicit `empty` store then re-applies its own + # seed_files on top — every clone/reset is where its declared initial state actually gets + # (re)established, since freeze deliberately skipped it. + try: + if data_dir.exists(): + shutil.rmtree(data_dir) + data_dir.mkdir(parents=True, exist_ok=True) + except OSError as exc: + # N9, p6-review-r2 (MAJOR): a bare wipe-and-recreate of this process/data-dir setup, same + # class as `spawn_managed_process`'s own boundary. + raise ProcessRuntimeError( + "reset", "spawn_failed", f"{process.name}: preparing {data_dir}: {exc}", + process=process.name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, runner=context.runner, + sync_run=context.sync_run, user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + # B4, p6-review-r1: fresh spawn over a wiped data directory — the CREATE DATABASE/apply_ + # store_seed below talk to it immediately. + _wait_for_store_ready(manifest, process, port=port, context=context) + if store is not None and strategy is BaselineStrategy.EMPTY: + dbname = f"w{world_index}" if process.engine is ManagedEngine.POSTGRES else "" + if process.engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="reset", process_name=process.name) + _call_sql( + context.sql_runner, stage="reset", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'CREATE DATABASE "{dbname}"', + ) + if context.bundle_dir is None: + # Same invariant `freeze_baseline` guards explicitly — a silent `Path()` (cwd) fallback + # here would resolve `seed_files` against the wrong directory instead of failing typed. + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + "SpawnContext.bundle_dir is required to re-seed an empty-strategy store", + process=process.name, + ) + apply_store_seed( + store, engine=process.engine, bundle_dir=context.bundle_dir, port=port, + dbname=dbname, credentials=credentials, process_name=process.name, + sync_run=context.sync_run, user=handle.uid, group=handle.gid, + rabbitmq_import=context.rabbitmq_import, + ) + return handle + + +def _reset_template_database( + *, host: str, port: int, credentials: EngineCredentials, world_db: str, template_db: str, + sql_runner: SqlRunner, process_name: str, +) -> None: + """§4.2 `template_database`: "drop + recreate the world's logical DB from the template." The + admin connection targets `postgres` (never `world_db` itself or the template — postgres + forbids `DROP`/`CREATE DATABASE` from inside the database being touched). Terminates other + backends on `world_db` first — a connection lingering from that world's own just-terminated + `source` processes is exactly what would otherwise block the `DROP`. + """ + admin = dict( + host=host, port=port, user=credentials.username, password=credentials.password, + dbname="postgres", + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, + statement=( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{world_db}' AND pid <> pg_backend_pid()" + ), + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, statement=f'DROP DATABASE IF EXISTS "{world_db}"', + ) + _call_sql( + sql_runner, stage="reset", process_name=process_name, + **admin, statement=f'CREATE DATABASE "{world_db}" TEMPLATE "{template_db}"', + ) + + +def _clone_or_reset_world( + manifest: EnvironmentBundleV2, + world_index: int, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + existing_handles: dict[str, SpawnedWorldProcess], +) -> WorldSpawnResult: + """Shared by `ProcessRuntimeProvider._ensure_world` (first creation / sick-world replace) and + `reset_world` (mid-job restore) — both are "terminate this world's own per-world handles, + reseal each managed store from the baseline, respawn `source` processes"; they differ only in + what the caller does afterward (a fresh/replaced world is left `preparing`, judged later by + `healthy()`'s declared readiness probes; a reset world is sentinel-checked immediately, per + §4.2, and marked `unhealthy` on failure). + """ + # N12, p6-review-r2 (MINOR): reversed — `existing_handles`' insertion order is `spawn_world`'s + # own `_topological_order` (dependencies first); terminating it forward sends SIGTERM to this + # world's own postgres while its `tools-api`/`agent` may still hold connections, guaranteeing + # the full escalation wait every reset. A dict preserves insertion order, so `reversed()` here + # IS reverse-topological order without recomputing it. + for name, handle in reversed(list(existing_handles.items())): + if name not in job_shared_handles: + # M7, p6-review-r1: waits for real exit before `_seal_world_store` below `rmtree`s + # this same process's data directory and rebinds its port — a bare `terminate()` + # racing that `rmtree` is exactly `EADDRINUSE` / "remove a live server's data dir". + _terminate_and_wait( + handle.handle, prefer_interrupt=_prefers_interrupt(manifest, name), + ) + + store_by_process = _store_by_process_name(manifest) + baseline_by_process = {record.process_name: record for record in baseline.stores} + new_handles: dict[str, SpawnedWorldProcess] = dict(job_shared_handles) + try: + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + # `_seal_world_store` runs even for an already-running job-shared engine + # (`process.name` already in `new_handles`, seeded from `job_shared_handles` above) + # — its own logical `wN` database still needs resetting every time, even though the + # shared ENGINE PROCESS itself does not; `template_database`'s branch returns `None`, + # so `new_handles` is left holding the pre-seeded shared handle untouched, exactly as + # intended. + store = store_by_process.get(process.name) + record = baseline_by_process.get(process.name) + sealed = _seal_world_store( + manifest, process, store, record, world_index=world_index, context=context, + ) + if sealed is not None: + new_handles[process.name] = sealed + except BaseException as exc: + # N4, p6-review-r2 (MAJOR): a raise partway through this loop (a bad seed_files entry on + # a LATER process, `_wait_for_store_ready` timing out) used to drop every EARLIER + # process's freshly-(re)sealed handle on the floor — nothing holds it, so `close()`'s own + # `rmtree` of this world's data directory next runs against a still-live engine. Attached + # to the exception (never terminated here) so the caller — which alone knows whether this + # is a first clone with nothing to lose, or a reset it may still want to retry — can merge + # it into its own live-handle bookkeeping before deciding what to do next. + exc.partial_handles = new_handles # type: ignore[attr-defined] + raise + + # `spawn_world` sets its own (more complete — it starts from `new_handles` and accumulates + # further) `partial_handles` on a raise, so no extra wrapping is needed here for that case. + return spawn_world(manifest, world_index=world_index, context=context, shared_handles=new_handles) + + +def _check_all_sentinels( + manifest: EnvironmentBundleV2, world_index: int, *, context: SpawnContext, +) -> bool: + """§4.2: "after every reset the store's sentinel must pass" — every declared store, not just + the first; a single failing sentinel fails the whole check (short-circuits nothing, so every + store is always attempted — useful diagnostics beat an early return here).""" + store_by_process = _store_by_process_name(manifest) + ok = True + for process in manifest.processes: + if not isinstance(process, ManagedProcess): + continue + store = store_by_process.get(process.name) + if store is None: + continue + port = context.port_plan.port_for(process.name, world_index) + dbname = f"w{world_index}" if process.engine is ManagedEngine.POSTGRES else None + passed = check_sentinel( + store, engine=process.engine, host="localhost", port=port, dbname=dbname, + credentials=context.credentials.get(process.name), sql_runner=context.sql_runner, + redis_runner=context.redis_runner, rabbitmq_inspector=context.rabbitmq_inspector, + process_name=process.name, stage="reset", + ) + ok = ok and passed + return ok + + +def reset_world( + manifest: EnvironmentBundleV2, + world_index: int, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + existing_handles: dict[str, SpawnedWorldProcess], +) -> tuple[dict[str, SpawnedWorldProcess], bool]: + """§4.2's per-world reset, exactly — returns the world's refreshed handle map and whether + every declared store's sentinel passed afterward. NEVER raises for a sentinel failure: §4.2's + own words, "a sentinel failure marks the world unhealthy," is the caller's job to act on, not + this function's to escalate. A genuine provisioning failure below this (a `ProcessRuntimeError` + from spawn/build) still raises — that is not what "sentinel failure" covers. + """ + result = _clone_or_reset_world( + manifest, world_index, context=context, baseline=baseline, + job_shared_handles=job_shared_handles, existing_handles=existing_handles, + ) + ok = _check_all_sentinels(manifest, world_index, context=context) + return result.handles, ok + + +# --- §4 conformance gate ------------------------------------------------------------------------- + +_CONFORMANCE_CANARY_NAME = "_alk_conformance" # §2c reserved name; mirrors process_preflight.py's +# own `_RESERVED_NAME` and `world/handle.py`'s `CONFORMANCE_TABLE` — redeclared locally rather +# than imported, matching the existing split of that same constant across those two modules. +_CONFORMANCE_MARKER = "alk-conformance-canary" + +_CANARY_PROTOCOL_PREFERENCE = ( + CapabilityProtocol.POSTGRES, CapabilityProtocol.REDIS, CapabilityProtocol.AMQP, +) + + +def _first_canary_store(manifest: EnvironmentBundleV2) -> StoreEntry | None: + """§4: "the first store by protocol preference postgres > redis > rabbitmq." `no_sql_store` + (§2e item 6) guarantees a `kind: process` bundle always has a postgres store, so `None` is + reachable only for the vacuous zero-store case §4 itself names (never actually a `kind: + process` bundle in practice).""" + if manifest.seed is None: + return None + by_protocol: dict[CapabilityProtocol, StoreEntry] = {} + for store in manifest.seed.stores: + capability = manifest.capabilities[store.capability] + by_protocol.setdefault(capability.protocol, store) + for protocol in _CANARY_PROTOCOL_PREFERENCE: + if protocol in by_protocol: + return by_protocol[protocol] + return None + + +def _engine_for_store(manifest: EnvironmentBundleV2, store: StoreEntry) -> ManagedEngine: + capability = manifest.capabilities[store.capability] + processes_by_name = {process.name: process for process in manifest.processes} + process = processes_by_name[capability.service] + if not isinstance(process, ManagedProcess): + raise ProcessRuntimeError( + "conformance", _INTERNAL_INVARIANT_VIOLATED, + f"{store.capability}: backing service is not a managed engine; bundle_v2's " + "store_service_not_managed should have rejected this bundle", + ) + return process.engine + + +def _run_canary_probe( + manifest: EnvironmentBundleV2, store: StoreEntry, engine: ManagedEngine, *, context: SpawnContext, +) -> bool: + """Creates the reserved object in world 0, then asserts it is NOT visible in world 1 — proving + the two worlds are really isolated from each other, not just reachable on different ports.""" + capability = manifest.capabilities[store.capability] + process_name = capability.service + credentials = context.credentials.get(process_name) + port0 = context.port_plan.port_for(process_name, 0) + port1 = context.port_plan.port_for(process_name, 1) + + if engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, user=credentials.username, password=credentials.password, + dbname="w0", + statement=( + f'CREATE TABLE "{_CONFORMANCE_CANARY_NAME}" (marker text); ' + f"INSERT INTO \"{_CONFORMANCE_CANARY_NAME}\" VALUES ('{_CONFORMANCE_MARKER}')" + ), + ) + rows = _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port1, user=credentials.username, password=credentials.password, + dbname="w1", statement=f"SELECT to_regclass('{_CONFORMANCE_CANARY_NAME}') IS NOT NULL", + read_only=True, # N8, p6-review-r2. + ) + return not (rows and rows[0] and rows[0][0]) + if engine is ManagedEngine.REDIS: + _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, + command=["SET", _CONFORMANCE_CANARY_NAME, _CONFORMANCE_MARKER], + ) + value = _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port1, command=["GET", _CONFORMANCE_CANARY_NAME], + ) + return value is None + if engine is ManagedEngine.RABBITMQ: + # rabbitmq is never job-shared (§2b: not in `_ENGINE_STRATEGIES`'s `template_database` + # set) — world 0 and world 1 are already separate broker processes on separate ports, so + # there is no shared state left to leak between them. Declaring the queue in world 0 and + # confirming world 1's own broker never saw it is what is left to prove — m8, p6-review- + # r1: previously only ever INSPECTED (a read), so world 1's check compared against a + # queue that had never been created anywhere, a vacuous pass regardless of isolation. + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + _call_rabbitmq_action( + context.rabbitmq_declare, stage="conformance", process_name=process_name, + action="declare", + host="localhost", port=port0, credentials=credentials, + queue=_CONFORMANCE_CANARY_NAME, message=_CONFORMANCE_MARKER, + ) + depth = _call_rabbitmq_with_retry( + context.rabbitmq_inspector, stage="conformance", process_name=process_name, + host="localhost", port=port1, credentials=credentials, queue=_CONFORMANCE_CANARY_NAME, + accept=lambda value: value in (0, None), + ) + # N15, p6-review-r2 (MINOR): no explicit delete here anymore. `run_conformance_gate`'s own + # `reset_world` calls for BOTH worlds run right after this returns, and world 0's rabbitmq + # reset (`datadir_copy`, its only legal strategy — §2b) wipes and restarts it from the + # pristine baseline snapshot regardless — the same mechanism postgres/redis already rely + # on for their own canary cleanup. Deleting HERE used to make `_verify_canary_absent`'s + # LATER read of this same queue vacuous: it was already gone from an earlier, unrelated + # step, proving nothing about whether the reset itself actually worked. A failed probe + # (isolation broken) still returns here without a reset ever running — the reserved name + # keeps that residual harmless (never collides with customer content). + return depth in (0, None) + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + +def _verify_canary_absent( + manifest: EnvironmentBundleV2, store: StoreEntry, engine: ManagedEngine, *, context: SpawnContext, +) -> bool: + """§4: "assert it is gone" — checked against world 0's OWN namespace after both worlds reset, + defense in depth against a reset that silently failed to actually seal a fresh baseline.""" + capability = manifest.capabilities[store.capability] + process_name = capability.service + credentials = context.credentials.get(process_name) + port0 = context.port_plan.port_for(process_name, 0) + + if engine is ManagedEngine.POSTGRES: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + rows = _call_sql( + context.sql_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, user=credentials.username, password=credentials.password, + dbname="w0", statement=f"SELECT to_regclass('{_CONFORMANCE_CANARY_NAME}') IS NOT NULL", + read_only=True, # N8, p6-review-r2. + ) + return not (rows and rows[0] and rows[0][0]) + if engine is ManagedEngine.REDIS: + value = _call_redis( + context.redis_runner, stage="conformance", process_name=process_name, + host="localhost", port=port0, command=["GET", _CONFORMANCE_CANARY_NAME], + ) + return value is None + if engine is ManagedEngine.RABBITMQ: + credentials = _require_credentials(credentials, stage="conformance", process_name=process_name) + depth = _call_rabbitmq_with_retry( + context.rabbitmq_inspector, stage="conformance", process_name=process_name, + host="localhost", port=port0, credentials=credentials, queue=_CONFORMANCE_CANARY_NAME, + accept=lambda value: value in (0, None), # N15, p6-review-r2. + ) + return depth in (0, None) + return False # pragma: no cover - ManagedEngine is closed; unreachable past the model layer. + + +def run_conformance_gate( + manifest: EnvironmentBundleV2, + *, + context: SpawnContext, + baseline: BuildOutput, + job_shared_handles: dict[str, SpawnedWorldProcess], + world_handles: dict[int, dict[str, SpawnedWorldProcess]], +) -> tuple[bool, str | None]: + """§4's 2-world canary, run once per attempt after baseline freeze: create the reserved + `_alk_conformance` object in world 0, assert it is invisible in world 1, `reset` both, assert + it is gone + every sentinel passes both worlds. TRULY never raises (M3, p6-review-r1) — every + gate-procedure check returns `(False, "conformance_gate_failed")`, and a `ProcessRuntimeError` + RAISED below this (B5's typed store-seam failures, `reset_world`'s own spawn/reseal path, a + missing-credentials invariant) is now caught and degrades the SAME way rather than failing the + whole job: §4's own words are "Fail -> effective parallelism 1 ... Loud, never silent," which + governs the GATE'S RESULT, not just a False return from its own checks — a world that has not + finished booting when the canary dials it (B4 narrows but does not eliminate this race) is an + isolation-boundary problem exactly like a failed canary, not a reason to fail the job. Requires + worlds 0 and 1 to already exist in `world_handles` — the caller (`ProcessRuntimeProvider`) + provisions a throwaway 2-world pair before calling this, then reconciles down to 1 on a + `False` return, or up to the job's real `W` on `True`. + """ + store = _first_canary_store(manifest) + if store is None: + return True, None # §2e's own no_sql_store guarantee means this never actually happens. + engine = _engine_for_store(manifest, store) + + world_index: int | None = None + try: + if not _run_canary_probe(manifest, store, engine, context=context): + return False, "conformance_gate_failed" + + for world_index in (0, 1): + handles, sentinel_ok = reset_world( + manifest, world_index, context=context, baseline=baseline, + job_shared_handles=job_shared_handles, + existing_handles=world_handles.get(world_index, {}), + ) + world_handles[world_index] = handles + if not sentinel_ok: + return False, "conformance_gate_failed" + + if not _verify_canary_absent(manifest, store, engine, context=context): + return False, "conformance_gate_failed" + return True, None + except (ProcessRuntimeError, OSError, shutil.Error) as exc: + # N9, p6-review-r2: widened past `ProcessRuntimeError` alone — `reset_world` below this + # runs `rmtree`/`copytree`/`chown`/`chmod` (`_seal_world_store`'s own work), and a bare + # filesystem fault there used to escape this "TRULY never raises" (M3) gate untyped. + # N4, p6-review-r2: a raise from `reset_world` (via `_clone_or_reset_world`) mid-loop + # carries whatever THIS world's own partial reseal produced (`exc.partial_handles`) — + # merged here so a live engine the gate's own reset just spawned is not left unreferenced + # in `world_handles` the moment the gate degrades instead of propagating the failure. + partial = getattr(exc, "partial_handles", None) + if partial is not None and world_index is not None: + world_handles[world_index] = partial + # The cause cannot travel in the returned `reason` string — that value is §5's own closed + # `parallelism_degraded.reason` vocabulary (`conformance_gate_failed` | `fixed_port`) — so + # "loud" means logged here, not encoded in the return. + if isinstance(exc, ProcessRuntimeError): + logger.error( + "conformance gate raised %s/%s: %s degrading to effective parallelism 1 instead " + "of failing the job", exc.stage, exc.code, exc, + ) + else: + logger.error( + "conformance gate raised %s: %s degrading to effective parallelism 1 instead of " + "failing the job", type(exc).__name__, exc, + ) + return False, "conformance_gate_failed" + + +# --- §0.3/§4 rule 1 secrets lifetime (B3, p6-review-r1) --------------------------------------- + + +def _read_job_secret_purposes(work_directory: Path) -> dict[str, str]: + """§0.2/§4.1: `/work/job.json` is the provisioner's own configuration source — `agent. + secret_refs` (§1: `{alias: {manager, key, version, purpose}}`) is where each alias's REAL + purpose comes from — never relabelled `target_provider` unconditionally, which would silently + defeat `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion the moment a + `source_checkout` alias ever reaches `secrets.json`. The two raises below use `spawn_failed` + for a malformed `job.json`, which is a vocabulary stretch — §2f's domain rule for it is + "infrastructure if a managed engine, `agent` if source," neither of which is what a + config-read fault actually is; picked as the closest §4.6 code available, domain + `infrastructure` (a corrupted upload, not a bundle-authoring fault the customer caused). + """ + job_json_path = work_directory / "job.json" + if not job_json_path.exists(): + # A real gap on the hosted path (§0.2 guarantees the file), not a local-lane default to + # paper over silently — the caller still proceeds (a constructor-supplied `secret_purpose_ + # map`, or the local/test lane, never needs this file at all), but every alias in + # `secrets.json` then has no purpose to match and is correctly dropped below, which would + # otherwise look like a silent injection failure three layers down with nothing in the + # log explaining why. + logger.warning( + "secrets: %s is absent; every alias in secrets.json will be dropped (no purpose to " + "match) unless a secret_purpose_map was supplied", job_json_path, + ) + return {} + try: + raw = json.loads(job_json_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProcessRuntimeError( + "secrets", "spawn_failed", f"{job_json_path}: unreadable or not valid JSON: {exc}", + domain=FailureDomain.INFRASTRUCTURE, + ) from exc + if not isinstance(raw, dict): + raise ProcessRuntimeError( + "secrets", "spawn_failed", + f"{job_json_path}: expected a JSON object, got {type(raw).__name__}", + domain=FailureDomain.INFRASTRUCTURE, + ) + agent = raw.get("agent") + refs = agent.get("secret_refs") if isinstance(agent, dict) else None + if not isinstance(refs, dict): + return {} + return { + str(alias): ref["purpose"] + for alias, ref in refs.items() + if isinstance(ref, dict) and isinstance(ref.get("purpose"), str) + } + + +def _load_and_delete_secrets( + secrets_path: Path, *, work_directory: Path, secret_purpose_map: dict[str, str] | None = None, +) -> tuple[dict[str, str], dict[str, str]]: + """§0 step 3 / §4 rule 1's lifetime rule: "the provisioner loads this file into memory at + startup and deletes it immediately after loading, BEFORE ANY CUSTOMER PROCESS STARTS. The + in-memory map lives for the whole job — `reset` restarts and `provision` reconciliations + re-inject from memory." + + Each alias's purpose comes from `job.json`'s own `agent.secret_refs` + (`_read_job_secret_purposes`) — NOT invented as `target_provider` for every alias, which would + retire `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion the moment a + `source_checkout` alias ever reached this file. `secret_purpose_map`, when given, overrides + the job.json read entirely (the local/test lane's own shape — no `/work/job.json` on a dev + box). An alias in `secrets.json` with no matching ref anywhere is dropped, not injected under + a guessed purpose, and logged — `select_process_secrets` naturally never matches an alias + absent from the purposes map, so nothing further is needed to enforce the drop. + """ + if not secrets_path.exists(): + values: dict[str, str] = {} + else: + try: + raw = json.loads(secrets_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProcessRuntimeError( + "secrets", "spawn_failed", f"{secrets_path}: unreadable or not valid JSON: {exc}", + domain=FailureDomain.INFRASTRUCTURE, + ) from exc + secrets_path.unlink(missing_ok=True) + if not isinstance(raw, dict): + raise ProcessRuntimeError( + "secrets", "spawn_failed", + f"{secrets_path}: expected a JSON object of alias -> value, got " + f"{type(raw).__name__}", + domain=FailureDomain.INFRASTRUCTURE, + ) + values = {str(alias): str(value) for alias, value in raw.items()} + + all_purposes = ( + dict(secret_purpose_map) if secret_purpose_map is not None + else _read_job_secret_purposes(work_directory) + ) + dropped = sorted(alias for alias in values if alias not in all_purposes) + if dropped: + logger.warning( + "secrets: %d alias(es) with no matching agent.secret_refs entry will never be " + "injected into any process: %s", len(dropped), dropped, + ) + purposes = {alias: all_purposes[alias] for alias in values if alias in all_purposes} + return values, purposes + + +# --- §4 provision / close: the stateful RuntimeProvider adapter ----------------------------------- + + +class ProcessRuntimeProvider: + """The §4 `RuntimeProvider`, structurally: a stateful adapter around this module's pure + functions (`spawn_world`, `freeze_baseline`, `reset_world`, `run_conformance_gate`, ...), + holding what a §3-shaped `EnvironmentRuntime` cannot carry itself — live process handles, the + job's generated credentials, and the sealed baseline. `runtime.py`'s own `RuntimeProvider` + Protocol is untouched (P5's docstring: "a later phase wires a Protocol-conforming, stateful + adapter around the pure functions below") — wiring the existing local-SDK callers onto this + shape is the refactor-sized "Implementation delta" §4 itself calls out, a separate change from + this one; `provision`/`reset`/`close` below match §4's method shapes (not `runtime.py`'s older + v1 ones — no `provider` field, `provision` takes `instances` and returns a list, `close` takes + no `runtime` argument), since those are what this phase actually implements. + + Idempotency (§4 rule 1, "idempotent for the job identity") holds for repeated calls against the + SAME instance — the normal in-job shape: the entrypoint constructs one provider per job and + calls `provision` once, then `reset`/`provision` again for sick-world recovery, all against + that one long-lived object for the guest process's whole life. Resuming after the GUEST + PROCESS ITSELF restarts (a fresh `hosted_entrypoint` invocation after a crash) would need + filesystem-observable state beyond what this class attempts — `write_build_output`'s own + `inputs_digest` reuse (§2c: "the provisioner records it... as the baseline identity for + attempt-retry reuse") is the one piece of that this module produces; resurrecting LIVE process + handles across a process boundary is the entrypoint's own concern, out of this phase's file + scope. + + n2, p6-review-r2: this class takes no lock over its own mutable state (`_world_handles`, + `_runtimes`, `_context`, `_secrets_loaded`, ...) behind the `asyncio.to_thread` calls below — + that is deliberate, not an oversight. v1.12 §4.5b makes the port NON-REENTRANT and puts + serialization on the SCHEDULER, never the provider: every `provision`/`reset`/`healthy`/ + `close` call for one job must be serialized by the caller — `healthy` demotes state, so it + writes and is in the set too (Q9, p6-review-r3). Do not add a lock here on the assumption one + is missing; add it at the call site instead. + """ + + name = "hosted-process" + + def __init__( + self, + *, + runner: ProcessRunner = default_process_runner, + sync_run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + prober: CapabilityProber = default_capability_prober, + sql_runner: SqlRunner = default_sql_runner, + redis_runner: RedisCommandRunner = default_redis_command_runner, + rabbitmq_inspector: RabbitmqQueueInspector = default_rabbitmq_queue_inspector, + rabbitmq_declare: RabbitmqQueueDeclarer = default_rabbitmq_queue_declare_and_publish, + rabbitmq_delete: RabbitmqQueueDeleter = default_rabbitmq_queue_delete, + rabbitmq_import: RabbitmqDefinitionsImporter = default_rabbitmq_definitions_importer, + copy: Callable[[Path, Path], None] | None = None, + user_resolver: Callable[[str], "pwd.struct_passwd | None"] = default_user_resolver, + chown: Callable[[Path, int, int], None] = _default_chown, + build_step_timeout_seconds: float = _DEFAULT_BUILD_STEP_TIMEOUT_SECONDS, + token: Callable[[], str] | None = None, + secrets_path: Path = Path("/run/futureagi/secrets.json"), + close_wait_timeout_seconds: float = _TERMINATE_WAIT_SECONDS, + secret_purpose_map: dict[str, str] | None = None, + ) -> None: + self._runner = runner + self._sync_run = sync_run + self._prober = prober + self._sql_runner = sql_runner + self._redis_runner = redis_runner + self._rabbitmq_inspector = rabbitmq_inspector + self._rabbitmq_declare = rabbitmq_declare + self._rabbitmq_delete = rabbitmq_delete + self._rabbitmq_import = rabbitmq_import + self._copy = copy + self._user_resolver = user_resolver + self._chown = chown + self._build_step_timeout_seconds = build_step_timeout_seconds + self._token = token + self._secrets_path = secrets_path + # N12, p6-review-r2: `close()`'s own bound on the per-handle wait/kill escalation — §0.7's + # 120s flush window is shared by every handle `_teardown_processes_and_directories` tears + # down, so a caller close to that deadline can pass something tighter than the module + # default without touching every OTHER termination call site's own timeout. + self._close_wait_timeout_seconds = close_wait_timeout_seconds + # N10, p6-review-r2: overrides the `/work/job.json` read entirely when supplied — the + # local/test lane's own shape (no job.json on a dev box); a hosted caller leaves this + # `None` and `_load_and_delete_secrets` reads `agent.secret_refs` itself. + self._secret_purpose_map = secret_purpose_map + + self._manifest: EnvironmentBundleV2 | None = None + self._bundle_digest: str | None = None + self._context: SpawnContext | None = None + self._build_output: BuildOutput | None = None + self._job_shared_handles: dict[str, SpawnedWorldProcess] = {} + self._world_handles: dict[int, dict[str, SpawnedWorldProcess]] = {} + self._runtimes: dict[int, EnvironmentRuntime] = {} + self._conformance_checked = False + # B3, p6-review-r1: loaded once (`_load_and_delete_secrets`), on the FIRST provision call + # for a job identity — `_secrets_loaded` latches so neither a later reconcile call nor a + # bundle-digest rebuild (M6) ever tries to re-read a file `close()`/the load itself has + # already deleted; §0.3's "re-inject from memory" is exactly these two dicts surviving on + # `self` across every `provision`/`reset` call for the object's whole life. + self._secret_values: dict[str, str] = {} + self._secret_purposes: dict[str, str] = {} + self._secrets_loaded = False + + async def provision( + self, + bundle: EnvironmentBundleV2, + *, + source: Path, + bundle_dir: Path, + work_directory: Path, + contract: Any | None = None, # accepted for §4 shape-compatibility; not consumed here — + # evidence-seam wiring is out of this phase's scope, same as `runtime.py`'s own providers. + instances: int = 1, + require_declared_user: bool = True, + ) -> list[EnvironmentRuntime]: + import asyncio + + return await asyncio.to_thread( + self._provision_sync, bundle, source=source, bundle_dir=bundle_dir, + work_directory=work_directory, instances=instances, + require_declared_user=require_declared_user, + ) + + def _provision_sync( + self, bundle: EnvironmentBundleV2, *, source: Path, bundle_dir: Path, work_directory: Path, + instances: int, require_declared_user: bool, + ) -> list[EnvironmentRuntime]: + port_plan = plan_ports(bundle, instances=instances) + effective = port_plan.effective_instances + bundle_digest = bundle.digest + + if self._manifest is None or self._bundle_digest != bundle_digest: + if self._manifest is not None: + # M6, p6-review-r1: a bundle-digest change used to reassign this instance's own + # identity fields straight over the PREVIOUS job's still-running processes and + # still-allocated ports — §4.1's "never duplicates" broken in the one case this + # branch exists for. Full teardown first (same mechanism `close()` uses, minus + # the secrets unlink — this is a re-sealed bundle, not a new job identity, so the + # in-memory secret map must survive it). + self._teardown_processes_and_directories(work_directory) + if not self._secrets_loaded: + # B3 / §0.3, p6-review-r1: loaded and the file deleted BEFORE any customer + # process starts — done here, before `build_process_trees`/`freeze_baseline` + # below ever spawn anything. N10, p6-review-r2: purposes come from `job.json`'s + # own `agent.secret_refs` (or the constructor override), never invented. + self._secret_values, self._secret_purposes = _load_and_delete_secrets( + self._secrets_path, work_directory=work_directory, + secret_purpose_map=self._secret_purpose_map, + ) + self._secrets_loaded = True + + # First call for this job identity, or the bundle changed underneath it (§2c: a + # digest mismatch forces a rebuild) — build once, seed+freeze once, gate once. + credentials = generate_engine_credentials(bundle, token=self._token) + context = SpawnContext( + work_directory=work_directory, port_plan=port_plan, credentials=credentials, + secret_values=self._secret_values, secret_purposes=self._secret_purposes, + runner=self._runner, sync_run=self._sync_run, + prober=self._prober, copy=self._copy, user_resolver=self._user_resolver, + require_declared_user=require_declared_user, chown=self._chown, + build_step_timeout_seconds=self._build_step_timeout_seconds, + sql_runner=self._sql_runner, redis_runner=self._redis_runner, + rabbitmq_inspector=self._rabbitmq_inspector, rabbitmq_declare=self._rabbitmq_declare, + rabbitmq_delete=self._rabbitmq_delete, rabbitmq_import=self._rabbitmq_import, + bundle_dir=bundle_dir, + ) + # N3, p6-review-r2 (MAJOR): the job identity (`_manifest`/`_bundle_digest`) is + # committed only AFTER `build_process_trees`/`freeze_baseline` both succeed — + # committing it first used to leave a failed first `provision()` claiming an identity + # with no build output, so an in-process retry (the same bundle, e.g. `hosted_ + # scheduler.py`'s background `_reconcile`) took the RECONCILE branch below instead of + # retrying the build, and hit the "context/build_output unset" invariant instead of + # the real, often-retryable cause. + try: + build_process_trees(bundle, source_root=source, context=context) + freeze_result = freeze_baseline( + bundle, bundle_digest=bundle_digest, context=context + ) + except (OSError, shutil.Error) as exc: + # §4.6 — filesystem failures during provisioning are `infrastructure`. This + # phase's own copy-heavy work (build trees, baseline snapshot/seal) can raise a raw + # filesystem error covering EITHER a source build tree or a managed baseline seal — + # `store_statement_failed` is a vocabulary stretch for a build-tree copy fault (the + # closest §2f code in a closed table with no generic `provisioner_io_failed`), but + # the domain is `infrastructure` either way, so it is set directly here rather than + # left to the fallback map. + self._manifest = None + self._bundle_digest = None + raise ProcessRuntimeError( + "baseline", "store_statement_failed", str(exc), + domain=FailureDomain.INFRASTRUCTURE, + ) from exc + except BaseException: + self._manifest = None + self._bundle_digest = None + raise + self._manifest = bundle + self._bundle_digest = bundle_digest + self._context = context + self._build_output = freeze_result.build_output + self._job_shared_handles = freeze_result.job_shared_handles + write_build_output(work_directory, self._build_output) + self._world_handles = {} + self._runtimes = {} + self._conformance_checked = False + else: + # Same job identity, a later reconcile call — `instances`/`require_declared_user` may + # legitimately differ (a sick-world recovery re-call), and so may `work_directory`/ + # `bundle_dir` (m7, p6-review-r1: the two used to silently diverge from whatever this + # call actually passed, since only `port_plan`/`require_declared_user` were carried + # forward here). The sealed baseline never re-runs regardless (§4 rule 1). + self._context = replace( + self._context, port_plan=port_plan, require_declared_user=require_declared_user, + work_directory=work_directory, bundle_dir=bundle_dir, + ) + + context = self._context + build_output = self._build_output + if context is None or build_output is None: + # m6, p6-review-r1: both branches above set these unconditionally — a bare `assert` + # here is stripped under `python -O`, which would turn a real "this is a bug in THIS + # function" precondition into an opaque `AttributeError` a few lines down instead of + # a typed failure. + raise ProcessRuntimeError( + "provision", _INTERNAL_INVARIANT_VIOLATED, + "context/build_output unset after the first-call/reconcile branch", + ) + + requested = instances + # At requested=1 nothing can degrade (effective=1 too, so no valid + # `parallelism_degraded` payload exists; outbound-channels.md's `1 <= effective < + # requested` bound is empty at requested=1). `port_plan.degraded_reason` is already + # None at instances=1, but that alone is not sufficient — see the + # `build_output.degrade_reason` write below, which is what actually enforces it + # against the conformance-gate paths that reassign `degrade_reason` further down. + degrade_reason = port_plan.degraded_reason if effective < requested else None + + if effective > 1 and not self._conformance_checked: + self._ensure_world(0) + self._ensure_world(1) + passed, reason = run_conformance_gate( + bundle, context=context, baseline=build_output, + job_shared_handles=self._job_shared_handles, world_handles=self._world_handles, + ) + build_output.conformance = passed + build_output.conformance_reason = reason + self._conformance_checked = True + if not passed: + effective = 1 + degrade_reason = reason + elif self._conformance_checked and build_output.conformance is False: + # A degrade decided by an EARLIER call must keep holding on every later reconcile call + # too — `effective` above is freshly recomputed from `port_plan` each time and knows + # nothing about a gate result from a call that already happened. + effective = 1 + degrade_reason = "conformance_gate_failed" + + build_output.requested_parallelism = requested + build_output.effective_parallelism = effective + # The conformance-gate branches above reassign `degrade_reason` unconditionally (they + # only know a gate result, not this call's `requested`) — re-checking the invariant here, + # at the single write site, covers those paths too instead of only the fixed_port input. + build_output.degrade_reason = degrade_reason if effective < requested else None + write_build_output(work_directory, build_output) + + # Reconcile down first — a prior call may have over-provisioned (the canary's own 2-world + # pair, before a gate failure dropped `effective` to 1). + for stale_index in [index for index in self._runtimes if index >= effective]: + self._teardown_world(stale_index) + for world_index in range(effective): + self._ensure_world(world_index) + # t3 / §4.1, p6-review-r1: "reconciles to exactly `instances` READY worlds" — + # `_ensure_world` only ever leaves a (re)built world `PREPARING` (§3's transition table + # promotes it later); promoted here via the same declared-readiness probe `healthy()` + # uses, so `provision()` itself returns worlds already at their reachable terminal state + # instead of leaving every caller to independently discover it must call `healthy()` + # first. A world left `READY`/`UNHEALTHY` by an earlier call is never re-probed here + # (mirrors `healthy()`'s own F6 "never promote except from PREPARING" rule). + for world_index in range(effective): + runtime = self._runtimes[world_index] + if runtime.state is RuntimeState.PREPARING: + # N2, p6-review-r2 (BLOCKER): polls (`_poll_runtime_health`), never samples once — + # nothing waits for the DAG's terminal process (`spawn_world` only waits on + # `depends_on` edges), so a single-shot probe here marked a bundle whose readiness- + # bearing process is terminal `UNHEALTHY` on every provision, regardless of how + # quickly it actually came up. + healthy_now = _poll_runtime_health(self._manifest, runtime, prober=context.prober) + runtime.state = RuntimeState.READY if healthy_now else RuntimeState.UNHEALTHY + + return [self._runtimes[index] for index in range(effective)] + + def _ensure_world(self, world_index: int) -> None: + """§4 rule 1: "completes or replaces partial/unhealthy worlds and never duplicates." A + world already `ready`/`preparing` is left alone (never duplicated); anything else (absent, + `unhealthy`, `stopped`) is (re)built from the sealed baseline via `_clone_or_reset_world` + — the same primitive `reset_world` uses, so a sick-world replace and a first-time clone are + one code path, not two. + """ + if self._manifest is None or self._context is None or self._build_output is None: + # m6, p6-review-r1: real precondition (only ever called from within `_provision_sync`, + # after all three are set) — typed, not a bare `assert` an `-O` run would strip. + raise ProcessRuntimeError( + "provision", _INTERNAL_INVARIANT_VIOLATED, + "_ensure_world called before manifest/context/build_output were established", + ) + existing = self._runtimes.get(world_index) + if existing is not None and existing.state in (RuntimeState.READY, RuntimeState.PREPARING): + return + + # N11, p6-review-r2. `current_world_index`: Q7, p6-review-r3 — a respawn reseals every + # tracked world's database, so every world OTHER than this one must be demoted for the + # scheduler to notice and repair it. + self._ensure_job_shared_handles_alive(current_world_index=world_index) + try: + result = _clone_or_reset_world( + self._manifest, world_index, context=self._context, baseline=self._build_output, + job_shared_handles=self._job_shared_handles, + existing_handles=self._world_handles.get(world_index, {}), + ) + except (OSError, shutil.Error) as exc: + # A raw filesystem fault this deep in `_clone_or_reset_world` (mkdir/chown/chmod for a + # (re)spawned process) does not by itself say which process's kind failed — `domain` + # is left unset so the §4 seam's fallback map applies (SECTION_2F_DOMAIN), since every + # raise site that DOES know its process kind already carries its own domain directly. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial # never orphan a live engine. + raise ProcessRuntimeError("spawn", "spawn_failed", str(exc)) from exc + except BaseException as exc: + # A raise partway through `_clone_or_reset_world`/`spawn_ + # world` used to drop every ALREADY-(re)sealed/spawned handle of THIS world on the + # floor — nothing held it, so `close()`'s own `rmtree` of this world's data directory + # next ran against a still-live server. `exc.partial_handles` (set by `freeze_ + # baseline`/`_clone_or_reset_world`/`spawn_world`'s own try/finally) is whatever this + # (re)build managed before failing; published here so the next reconcile/close can + # terminate it instead. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial + raise + + self._world_handles[world_index] = result.handles + # N1, p6-review-r2 (BLOCKER): MUTATES the existing `EnvironmentRuntime` in place rather + # than constructing a replacement — v1.12 §4.5b's live-object model ("providers hand out + # live `EnvironmentRuntime` objects") reads as ONE object per world for the provider's + # whole life. Minting a new object every rebuild meant `reset()`'s own state write (m5) + # landed on an object no caller who captured an EARLIER reference would ever see again. + metadata = _dispatch_metadata(result.handles) + if existing is not None: + existing.runtime_id = new_runtime_id(self._bundle_digest, world_index) + existing.endpoints = result.endpoints + existing.state = RuntimeState.PREPARING + existing.metadata = metadata + self._runtimes[world_index] = existing + else: + self._runtimes[world_index] = EnvironmentRuntime( + runtime_id=new_runtime_id(self._bundle_digest, world_index), + world_index=world_index, bundle_digest=self._bundle_digest, + state=RuntimeState.PREPARING, endpoints=result.endpoints, + metadata=metadata, + ) + + def _drop_world_shared_databases(self, world_index: int) -> None: + """m4, p6-review-r1: reconciling W down (e.g. after a conformance-gate degrade from 3 to + 1) used to leave `w1`/`w2` behind forever on a job-shared `template_database` engine — + nothing ever DROPs a world's logical DB on teardown, only on REUSE (`_seal_world_store`'s + own `IF EXISTS`). A space leak, not a correctness one; best-effort (logged, not raised) so + a teardown-time drop failure never blocks the reconcile it is cleaning up after. + """ + if self._manifest is None or self._context is None: + return + store_by_process = _store_by_process_name(self._manifest) + for process in self._manifest.processes: + if not isinstance(process, ManagedProcess) or process.engine is not ManagedEngine.POSTGRES: + continue + store = store_by_process.get(process.name) + if store is None or store.baseline.strategy is not BaselineStrategy.TEMPLATE_DATABASE: + continue + if process.name not in self._job_shared_handles: + continue + credentials = self._context.credentials.get(process.name) + if credentials is None: + continue + port = self._context.port_plan.port_for(process.name, world_index) + world_db = f"w{world_index}" + try: + # N16, p6-review-r2 (MINOR): mirrors `_reset_template_database`'s own sibling + # call — a lingering backend on `world_db` (e.g. a scenario connection that never + # closed) otherwise blocks this DROP exactly the way it would block a reuse-time + # one, silently re-opening the space leak m4 closed. + _call_sql( + self._context.sql_runner, stage="teardown", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + f"WHERE datname = '{world_db}' AND pid <> pg_backend_pid()" + ), + ) + _call_sql( + self._context.sql_runner, stage="teardown", process_name=process.name, + host="localhost", port=port, user=credentials.username, + password=credentials.password, dbname="postgres", + statement=f'DROP DATABASE IF EXISTS "{world_db}"', + ) + except ProcessRuntimeError as exc: + logger.warning("teardown: failed to drop %s on %s: %s", world_db, process.name, exc) + + def _teardown_world(self, world_index: int) -> None: + handles = self._world_handles.pop(world_index, {}) + # N12, p6-review-r2 (MINOR): reversed — `handles`' insertion order is `spawn_world`'s own + # `_topological_order` (dependencies first), so terminating it forward sends SIGTERM to a + # per-world engine while its own dependents (`tools-api`/`agent`) may still hold open + # connections, guaranteeing the full escalation wait every time. A dict's insertion order + # is preserved, so `reversed()` here IS reverse-topological order, no recomputation needed. + for name, handle in reversed(list(handles.items())): + if name not in self._job_shared_handles: + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait(handle.handle, prefer_interrupt=prefer_interrupt) + self._drop_world_shared_databases(world_index) # m4 + runtime = self._runtimes.pop(world_index, None) + if runtime is not None: + runtime.state = RuntimeState.STOPPED + if self._context is not None: + # m4, p6-review-r1: the per-world scratch tree (`/work/worlds/w/`) covers every + # process under this world, managed and source alike — left behind otherwise, the + # other space-leak half of the same finding. + world_dir = self._context.work_directory / "worlds" / f"w{world_index}" + if world_dir.exists(): + shutil.rmtree(world_dir, ignore_errors=True) + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + import asyncio + + await asyncio.to_thread(self._reset_sync, runtime) + + def _reset_sync(self, runtime: EnvironmentRuntime) -> None: + if self._manifest is None or self._context is None or self._build_output is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, "reset() called before provision()", + ) + world_index = runtime.world_index + self._ensure_job_shared_handles_alive(current_world_index=world_index) # N11/Q7. + try: + handles, sentinel_ok = reset_world( + self._manifest, world_index, context=self._context, baseline=self._build_output, + job_shared_handles=self._job_shared_handles, + existing_handles=self._world_handles.get(world_index, {}), + ) + except (OSError, shutil.Error) as exc: + # reset's own filesystem work is fundamentally "reseal this world's stores from + # baseline" — the infrastructure-domain half of the store-statement mapping. + partial = getattr(exc, "partial_handles", None) + if partial is not None: + self._world_handles[world_index] = partial + raise ProcessRuntimeError( + "reset", "store_statement_failed", str(exc), domain=FailureDomain.INFRASTRUCTURE, + ) from exc + except BaseException as exc: + partial = getattr(exc, "partial_handles", None) # N4, p6-review-r2. + if partial is not None: + self._world_handles[world_index] = partial + raise + self._world_handles[world_index] = handles + # m5, p6-review-r1: mutates THIS PROVIDER's own current record for `world_index`, never + # blindly overwrites it with whatever `EnvironmentRuntime` the caller happened to pass in + # — a caller holding a STALE object (e.g. one `_ensure_world` already replaced with a + # fresh `runtime_id`, before this `reset()` call was even made) used to be able to stomp + # the live record with stale endpoints/id. N1, p6-review-r2: after `_ensure_world` mutates + # in place instead of replacing, the provider's record and every caller's own reference + # for this world index are the SAME object for the provider's whole life — the fallback + # below (constructing from the caller's own `runtime`) is kept only for the unreachable + # case where this provider somehow has no record for the index at all. + current = self._runtimes.get(world_index) + if current is None: + current = runtime + self._runtimes[world_index] = current + if not sentinel_ok: + current.state = RuntimeState.UNHEALTHY + return + # M4, p6-review-r1: §4 point 3 — "`healthy` = declared `readiness` probes, not 'process is + # running.'" A passing sentinel alone used to be enough to mark `READY`; nothing waited for + # the LAST process in the world's `depends_on` DAG (nobody depends on the control service, + # so `spawn_world` never probes it) — the scheduler could dispatch a scenario against an + # agent still mid-boot. N2, p6-review-r2: polls (`_poll_runtime_health`), the same helper + # `provision()`'s own promotion uses, instead of sampling `probe_runtime_health` once. + healthy_now = _poll_runtime_health(self._manifest, current, prober=self._context.prober) + current.state = RuntimeState.READY if healthy_now else RuntimeState.UNHEALTHY + + async def close(self, *, work_directory: Path) -> None: + import asyncio + + await asyncio.to_thread(self._close_sync, work_directory) + + def _teardown_processes_and_directories(self, work_directory: Path) -> None: + """Shared by `close()` and M6's bundle-digest-change rebuild — terminates every live + handle and removes every job/world-scoped directory WITHOUT touching secrets or this + instance's own identity fields; `close()` clears those itself right after calling this, + while a digest-change rebuild is about to overwrite them with the new bundle's own values + a few lines later in `_provision_sync`. + + m10, p6-review-r1: every step is individually guarded — a failure removing ONE directory + (a permission error, a file still open) used to abort the whole method before the LATER + steps (clearing this instance's own dicts) ever ran, so the second `close()` §4.4 requires + to be a no-op was no longer one. Logged, never raised, so cleanup always reaches the end. + """ + # N12, p6-review-r2 (MINOR): reversed per world (dependents before the engine they depend + # on — see `_teardown_world`'s own note) and bounded by `self._close_wait_timeout_seconds` + # so `close()` can keep itself inside §0.7's 120s flush window regardless of how many + # per-world engines it has to wait out. + for handles in self._world_handles.values(): + for name, handle in reversed(list(handles.items())): + if name not in self._job_shared_handles: + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait( # M7: real wait, not just a sent signal. + handle.handle, timeout=self._close_wait_timeout_seconds, + prefer_interrupt=prefer_interrupt, + ) + for name, handle in reversed(list(self._job_shared_handles.items())): + prefer_interrupt = self._manifest is not None and _prefers_interrupt( + self._manifest, name + ) + _terminate_and_wait( + handle.handle, timeout=self._close_wait_timeout_seconds, + prefer_interrupt=prefer_interrupt, + ) + for runtime in self._runtimes.values(): + runtime.state = RuntimeState.STOPPED + + for directory_name in ("build", "worlds", "managed"): + directory = work_directory / directory_name + try: + if directory.exists(): + shutil.rmtree(directory) + except OSError: + logger.warning("teardown: failed to remove %s; continuing", directory) + + self._job_shared_handles = {} + self._world_handles = {} + self._runtimes = {} + self._conformance_checked = False + + def _close_sync(self, work_directory: Path) -> None: + """§4 rule 4: idempotent hard-clean of everything — processes, data directories, build + trees, `secrets.json` if still present (the load in `_provision_sync` already deleted it + on the normal path; this is the "if still present" backstop §4.4 itself names, e.g. a + `close()` called before `provision()` ever ran). Idempotent by construction: every dict is + cleared at the end regardless of what failed along the way (m10). + """ + self._teardown_processes_and_directories(work_directory) + try: + self._secrets_path.unlink(missing_ok=True) # m9: no separate check-then-act .exists(). + except OSError: + logger.warning("close(): failed to unlink %s; continuing", self._secrets_path) + + self._manifest = None + self._bundle_digest = None + self._context = None + self._build_output = None + # Full reset, not just the spawn-state dicts `_teardown_processes_and_directories` + # already cleared — a provider reused for a genuinely NEW job after `close()` must load + # ITS OWN secrets file fresh, not silently run with the previous job's in-memory map (or + # none at all, since `_secrets_loaded` would otherwise still read `True`). + self._secret_values = {} + self._secret_purposes = {} + self._secrets_loaded = False + + def _ensure_job_shared_handles_alive(self, *, current_world_index: int | None = None) -> None: + """N11, p6-review-r2 (MAJOR): a job-shared `template_database` engine (§2b) backs EVERY + world at once, and nothing anywhere ever checked whether it was still running before + reusing it (`spawn_world`'s own `if name in handles: continue`) — an OOM-killed shared + postgres used to be carried forward as if healthy for the rest of the job, so every + subsequent `reset()`/reconcile raised `store_statement_failed` out of `_reset_template_ + database`'s own connection attempt against a dead process, `hosted_scheduler.py` swallowed + it, and no world ever recovered even though the engine's data directory was sitting intact + on disk the whole time. Called before `_clone_or_reset_world`/`reset_world` ever touch + `self._job_shared_handles`. `current_world_index` (Q7, p6-review-r3) is the world the + caller is already about to reconcile — passed through so a respawn can demote every OTHER + tracked world instead of leaving it `READY` against a database it just reset underneath. + """ + for name, handle in list(self._job_shared_handles.items()): + if not handle.handle.is_running(): + self._job_shared_handles[name] = self._respawn_dead_job_shared_engine( + name, current_world_index=current_world_index, + ) + + def _respawn_dead_job_shared_engine( + self, process_name: str, *, current_world_index: int | None = None, + ) -> SpawnedWorldProcess: + """Respawns from the SAME surviving data directory (never wiped — this is not a baseline + restore; the engine's own on-disk state is exactly what every world's logical database + still depends on), waits for readiness, then re-seals every world THIS PROVIDER currently + tracks from the template — the shared engine restarting is otherwise indistinguishable + from a fresh boot to postgres, but each world's own `w` database is a SEPARATE logical + database on that one instance and must be re-verified against the template again, same as + `_reset_template_database` already does for one world at a time. + """ + if self._manifest is None or self._context is None or self._build_output is None: + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + "_respawn_dead_job_shared_engine called before manifest/context/build_output " + "were established", + ) + processes_by_name = {process.name: process for process in self._manifest.processes} + process = processes_by_name.get(process_name) + if not isinstance(process, ManagedProcess): + raise ProcessRuntimeError( + "reset", _INTERNAL_INVARIANT_VIOLATED, + f"{process_name}: job-shared handle names a process that is not a managed engine", + process=process_name, + ) + context = self._context + build_output = self._build_output + port = context.port_plan.port_for(process_name, 0) + data_dir = managed_engine_data_dir(context.work_directory, process_name, world_index=None) + credentials = context.credentials.get(process_name) + try: + new_handle = spawn_managed_process( + process, port=port, data_dir=data_dir, credentials=credentials, + runner=context.runner, sync_run=context.sync_run, + user_resolver=context.user_resolver, + require_declared_user=context.require_declared_user, chown=context.chown, + ) + _wait_for_store_ready(self._manifest, process, port=port, context=context) + if process.engine is ManagedEngine.POSTGRES: + record = next( + (r for r in build_output.stores if r.process_name == process_name), None + ) + if record is not None: + live_credentials = _require_credentials( + credentials, stage="reset", process_name=process_name + ) + for world_index in self._runtimes: + _reset_template_database( + host="localhost", port=port, credentials=live_credentials, + world_db=f"w{world_index}", template_db=record.baseline_reference, + sql_runner=context.sql_runner, process_name=process_name, + ) + # Q7, p6-review-r3: the loop above just reset EVERY tracked world's database — + # demote every world other than the one the caller is already reconciling so + # the scheduler's next `reset()` repairs it, instead of handing out a `READY` + # world whose pooled connections point at a database dropped underneath it. + for idx, rt in self._runtimes.items(): + if idx != current_world_index: + rt.state = RuntimeState.UNHEALTHY + except (ProcessRuntimeError, OSError, shutil.Error) as exc: + raise ProcessRuntimeError( + "reset", "store_statement_failed", + f"{process_name}: job-shared engine died and could not be respawned: {exc}", + process=process_name, domain=FailureDomain.INFRASTRUCTURE, + ) from exc + return new_handle + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + """v1.12 §4.3's `RuntimeProvider.healthy` port, closing the gap p6-review-r2's task 2c + flagged: this class had `provision`/`reset`/`close` but no `healthy`, so it could not + structurally satisfy `runtime.py`'s `RuntimeProvider` Protocol. Demote-only, unlike the + module-level `healthy()` above — §3's transition table makes `unhealthy->ready` reachable + ONLY through a re-provision reconcile, and `provision()` already owns the `preparing-> + ready` promotion (`_poll_runtime_health`, before ever returning a world), so this port + method must never promote from ANY state, including `preparing`. `work_directory` is + accepted for §4 shape-compatibility and unused — the provider already holds its own + paths, same as `provision`'s own `contract` parameter. + + After N1 (p6-review-r2), `self._runtimes[world_index]` IS the same object every caller + holds for that world, so a single assignment on it is enough; `.get(..., runtime)` still + falls back to the caller's own object if this provider somehow has no record for the + index (unreachable in the normal `provision` -> `healthy` flow). + """ + import asyncio + + if self._manifest is None: + raise ProcessRuntimeError( + "healthy", _INTERNAL_INVARIANT_VIOLATED, "healthy() called before provision()", + ) + target = self._runtimes.get(runtime.world_index, runtime) + is_healthy = await asyncio.to_thread( + probe_runtime_health, self._manifest, target, prober=self._prober, + ) + if not is_healthy: + target.state = RuntimeState.UNHEALTHY + return is_healthy + + +__all__ = [ + "BuildOutput", + "CapabilityProber", + "EngineCredentials", + "EnvironmentRuntime", + "FreezeResult", + "PopenProcess", + "PortPlan", + "ProcessRunner", + "ProcessRuntimeError", + "ProcessRuntimeProvider", + "RabbitmqDefinitionsImporter", + "RabbitmqQueueDeclarer", + "RabbitmqQueueDeleter", + "RabbitmqQueueInspector", + "RedisCommandRunner", + "RuntimeEndpoint", + "RuntimeState", + "SpawnContext", + "SpawnedProcess", + "SpawnedWorldProcess", + "SqlRunner", + "StoreBaselineRecord", + "WorldSpawnResult", + "apply_seed_file", + "apply_store_seed", + "build_endpoints", + "build_process_tree", + "build_process_trees", + "build_tree_dir", + "check_sentinel", + "configuration_addresses_from_endpoints", + "default_capability_prober", + "default_process_runner", + "default_rabbitmq_definitions_importer", + "default_rabbitmq_queue_declare_and_publish", + "default_rabbitmq_queue_delete", + "default_rabbitmq_queue_inspector", + "default_redis_command_runner", + "default_sql_runner", + "default_user_resolver", + "freeze_baseline", + "generate_engine_credentials", + "healthy", + "managed_engine_data_dir", + "new_runtime_id", + "plan_ports", + "postgres_bootstrap_argv", + "postgres_daemon_argv", + "postgres_seed_argv", + "postgres_seed_env", + "probe_runtime_health", + "rabbitmq_conf_text", + "rabbitmq_daemon_argv", + "rabbitmq_daemon_env", + "rabbitmq_enabled_plugins_text", + "redis_daemon_argv", + "redis_seed_argv", + "render_capability_address", + "render_environment", + "render_template", + "reset_world", + "run_conformance_gate", + "select_process_secrets", + "spawn_managed_process", + "spawn_source_process", + "spawn_world", + "wait_for_dependency", + "world_scratch_dir", + "write_build_output", +] diff --git a/src/fi/alk/harness/scenario_source.py b/src/fi/alk/harness/scenario_source.py new file mode 100644 index 00000000..80455f40 --- /dev/null +++ b/src/fi/alk/harness/scenario_source.py @@ -0,0 +1,513 @@ +"""The scenario-source adapter: reads generated scenario documents out of the bundle (code-as-text, +on the on-disk layout `folder.py` documents) and turns them into the `Scenario`/`SubGoal` objects +`hosted_scheduler.py` actually drives. + +Deliberately does NOT import `fi.alk.harness.folder` or `fi.alk.harness.scenario` for the model: +both exist at HEAD, but HEAD's `Scenario` carries no `scenario_key`/`scenario_id` (those are +pr63-only) and its default `extra="ignore"` would silently discard exactly the two fields the +scheduler needs off a `scenario.json` written in the newer shape. So this module reads +`scenario.json` as a plain dict and pulls fields out by key, mirroring the documented layout +instead of depending on either model -- see the report's design-decisions section for the +consequences of that choice (HEAD-model drift). + +RESOLVED (p13-worker-r2, reports/p13-worker-r2.md CONTRACT NOTES): the `provision`/`begin` wire +shapes below follow the platform's actual, live route (futureagi/simulate/serializers/services/ +views `hosted_harness.py`) rather than Karthik's Scenario Generation Contract text (PR #63), where +the two disagree -- a single `POST .../scenarios/` discriminated by a body-level `operation` field, +`begin` keyed on the full `scenario_keys` set, and a provision response KEYED by `scenario_key` +(never a position-ordered array). `register_with_platform` below is the seam that builds those +payloads and merges the platform-assigned `scenario_id`s back onto each scenario. +""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING, Any, Callable, Sequence + +from . import outbound as ob +from .job import FailureDomain + +if TYPE_CHECKING: + from .hosted_entrypoint import ScenariosClient + +# LAYOUT DECISION (contract-silent -- hosted-execution-seams.md v1.15 §2 never mentions scenario +# documents, and §7 assigns the on-disk layout to Karthik's contract, status "in review"). Scenario +# documents live at `///...`, matching `folder.py`'s own +# `SCENARIOS` constant, so a write_folder destination of `` lands correctly with no +# translation. Kept as one module-level constant so a later contract can move it in one edit. +SCENARIOS_DIRNAME = "scenarios" + +_CHECKS_DIRNAME = "checks" +_SCENARIO_JSON = "scenario.json" +_SETUP_PY = "setup.py" +_READY_PY = "ready.py" + +# R1-5: divergence (a) (compile once, at load) moves every scenario file's compile+exec into +# `asyncio.to_thread(load_scenarios, ...)`, OUTSIDE every phase budget the scheduler enforces +# (`SETUP_TIMEOUT_SECONDS`/`READY_TIMEOUT_SECONDS`/`CHECK_TIMEOUT_SECONDS` all apply downstream, to +# `_run_phase`). Pathological module-level code (`while True: pass`, a blocking socket read) would +# otherwise hang the load with zero terminal events, no timeout catching it, ever. One generous +# wall-clock budget here converts that hang into a typed terminal instead -- a worker thread cannot +# actually be killed, so this accepts a leaked thread over an unbounded one, on the reasoning that a +# terminal event today is strictly better than none ever. +_LOAD_TIMEOUT_SECONDS = 60.0 + +# The TEXT of a judged sub-goal's check is never persisted by `folder.py`'s `write_folder` (only +# `SubGoal.deterministic()` entries get a `checks/.py` file) -- this fixed marker stands in +# for it so `SubGoal.judged` (mandatory; read by plain attribute access in `hosted_scheduler.py`) +# is never empty for a sub-goal this reader knows is judged. The round-trip loss this represents is +# recorded under CONTRACT QUESTIONS in the report. +_JUDGED_MARKER = "judged (reason not persisted by folder.py's on-disk layout)" + + +class ScenarioDocumentInvalid(RuntimeError): + """A scenario folder under `/scenarios/` is unreadable or malformed: missing + `scenario.json`, invalid JSON, a field of the wrong shape, or a `setup.py`/`ready.py`/ + `checks/.py` that will not compile. Raised rather than skipped -- `folder.py`'s own + `read_all` swallows exactly this and continues, which is right for a human editing a suite by + hand and wrong for a hosted job, where a bad scenario silently vanishing from the run reads as + a suite that passed with fewer scenarios than it should have.""" + + +def bundle_has_scenarios(bundle_dir: Path) -> bool: + """The LAYOUT DECISION's presence test: `/scenarios/` exists and at least one of + its subdirectories holds a `scenario.json`. Deliberately narrow -- an empty or missing + `scenarios/` must not flip the wiring decision away from the safe `NotWiredScenarioSource` + default. A bundle that means to carry scenarios but got the layout wrong fails loudly once + `load_scenarios` actually reads it, not silently by being treated as scenario-free here. + + An unreadable `scenarios/` directory (permission denied, race with deletion, etc.) reports + False rather than raising `OSError` (R1-1): this call sits in `hosted_entrypoint.py`'s wiring + `if` BEFORE the `try`/`except` that maps `ScenarioDocumentInvalid` to a typed terminal, so an + escape here would kill the whole guest process with no terminal event at all. Falling back to + `NotWiredScenarioSource`'s existing typed failure is the safe direction -- the alternative of + raising here has nowhere typed to land. + """ + root = bundle_dir / SCENARIOS_DIRNAME + if not root.is_dir(): + return False + try: + children = list(root.iterdir()) + except OSError: + return False + return any((child / _SCENARIO_JSON).is_file() for child in children if child.is_dir()) + + +def _judged_placeholder_check(world: Any, calls: Any) -> None: + """The check for a judged (non-deterministic) sub-goal: always reports "held", via the same + convention a deterministic check uses. `SubGoalResult.judged` -- not this return value -- is + what has to tell downstream a real judge still has to run; see CONTRACT QUESTIONS in the + report for the gap that leaves (a judged-only scenario reports "passed" before any judge runs). + """ + del world, calls + return None + + +def _compile_entry( + source: str, *, label: str, entry: str, allow_empty: bool = True +) -> Callable[..., object]: + """One scenario code-text -> a bare callable that raises, compiled ONCE here rather than per + call. Mirrors `folder.py`'s `_run` in exactly two respects: `compile(source, name, "exec")` + into a fresh, empty-dict namespace with default builtins, and (when `allow_empty`) + empty/whitespace-only source is a no-op success. Deliberately diverges from `_run` in the two + respects the brief calls out: + (a) compiling here, at load, turns a syntax error into one typed terminal for the whole job + instead of a per-scenario fault discovered mid-run; (b) the compiled function is returned + BARE, never wrapped in `_run`'s `Outcome` -- `hosted_scheduler.py`'s `_run_phase` is what + classifies a raised exception into `setup_crashed`/`ready_broken`/`check_broken`/a timeout, and + an `Outcome` return here would swallow every one of those before `_run_phase` ever saw it. + `_run`'s complaint-sentence return convention (a non-None, non-True, non-empty-string value + means "did not hold") is left untranslated for the same reason: `hosted_scheduler.py`'s own + `_classify_ready`/`_classify_check` already own that classification on the return-VALUE side of + this boundary; only the raise-vs-return boundary belongs to this module. + + `allow_empty=False` is for `check` entries only (R1-2): an EXISTING `checks/.py` that is + empty or whitespace-only compiles to nothing, and handing back a no-op "held" callable -- the + right behavior for "no setup/ready code here" -- would silently turn "there is no check" into + "the check passed": a vacuous deterministic pass, which is exactly what `hosted_scheduler.py` + forbids one scenario level up ("scenario declared zero sub_goals"). Absence of the file + entirely is what means "judged" (see `_load_one`); an existing-but-empty file is malformed. + """ + if not source.strip(): + if allow_empty: + return lambda *args: None + raise ScenarioDocumentInvalid(f"{label} defines no {entry}()") + try: + code = compile(source, f"<{label}>", "exec") + except (SyntaxError, ValueError) as exc: + # SyntaxError is the common case; a NUL byte in the source raises ValueError on some + # interpreter versions (R1-1) rather than SyntaxError -- both are the same content defect. + raise ScenarioDocumentInvalid(f"{label} would not compile: {exc}") from exc + namespace: dict[str, Any] = {} + try: + exec(code, namespace) # noqa: S102 - scenario code is meant to be exec'd; see CONTRACT QUESTIONS + except (Exception, SystemExit, KeyboardInterrupt) as exc: # noqa: BLE001 - see R1-1 + # A module-level `sys.exit()`/`raise SystemExit(...)` in the file itself is a malformed + # document, not a request to shut the guest process down -- `SystemExit`/`KeyboardInterrupt` + # are `BaseException`, not `Exception`, so a bare `except Exception` (the pre-R1-1 shape) + # let them straight through this boundary and out of `run_job` with zero terminal events, + # the guest exiting with whatever code the scenario file itself chose. + raise ScenarioDocumentInvalid(f"{label} would not compile: {exc}") from exc + function = namespace.get(entry) + if not callable(function): + raise ScenarioDocumentInvalid(f"{label} defines no {entry}()") + return function + + +@dataclass(frozen=True) +class _CompiledSubGoal: + """Satisfies `hosted_scheduler.SubGoal`: `name`/`judged` as plain attributes, `check` as a bare + callable. Stored as instance DATA rather than a `def check(self, world, calls)` method so + `goal.check(world, calls)` invokes the compiled function directly with exactly the two + positional arguments `_run_phase` passes -- a real method would prepend `self` as a third.""" + + name: str + judged: str + check: Callable[[Any, Any], object] + + +@dataclass(frozen=True) +class _CompiledScenario: + """Satisfies `hosted_scheduler.Scenario`. `scenario_key`/`scenario_id` are carried VERBATIM + from the document, including an empty `scenario_id` -- synthesizing one here would hide that + pre-allocation has not actually run (see CONTRACT QUESTIONS: receipts carry `scenario_id ""` + until that seam is wired). `setup`/`ready` are likewise stored as data, for the same reason as + `_CompiledSubGoal.check` above.""" + + scenario_key: str + scenario_id: str + sub_goals: tuple[_CompiledSubGoal, ...] + setup: Callable[[Any], object] + ready: Callable[[Any], object] + + +def _read_text(path: Path, *, label: str) -> str: + """Missing is "" (mirrors `folder.py`'s own missing-setup/ready-is-empty convention); present + but unreadable (permission denied, a directory instead of a file) or present but not valid + UTF-8 is a typed `ScenarioDocumentInvalid`, never a raw `OSError`/`UnicodeDecodeError` escaping + this module (R1-1) -- both are equally "this scenario folder is malformed", the same + conclusion `_load_one`'s other reads already reach for a bad `scenario.json`. + """ + if not path.exists(): + return "" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise ScenarioDocumentInvalid(f"{label}: cannot read {path.name}: {exc}") from exc + + +def _validate_subgoal_name(name: str, *, folder_name: str) -> None: + """R1-3: `sub_goals[]` entries are used verbatim to build `checks/.py` -- a path + separator or a `..` segment lets a name escape `checks/` (and the sealed bundle) entirely: an + absolute name execs an arbitrary file never hashed into the bundle's `files[]` (bypassing the + §2e integrity seal), and a `../`-style traversal that resolves to nothing silently turns into a + JUDGED sub-goal (`check_path.is_file()` is False) instead of a typed failure. Rejecting + anything but a plain filename component closes both.""" + if not name or "/" in name or "\\" in name or name in (".", ".."): + raise ScenarioDocumentInvalid( + f"{folder_name}: sub_goals name {name!r} is not a plain filename " + "(no path separators, no '..', no leading '/')" + ) + + +def _load_one(folder: Path) -> _CompiledScenario: + """One scenario folder -> a `Scenario`-protocol object. Mirrors `folder.py`'s documented + layout (`scenario.json` + `setup.py` + `ready.py` + `checks/.py`) but reads + `scenario.json` itself as a plain dict rather than through `fi.alk.harness.scenario.Scenario` + -- see the module docstring. `folder.py`'s `read_folder` restores only `setup_code`/ + `ready_code` from a folder; it does not read `checks/` at all, so every `checks/.py` for + each name in the document's `sub_goals` is read here, by this module, directly. + """ + body_path = folder / _SCENARIO_JSON + try: + raw = body_path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + # UnicodeDecodeError is a ValueError, not an OSError -- widened alongside it (R1-1) so a + # non-UTF-8 `scenario.json` is the same typed failure as an unreadable one, not an escape. + raise ScenarioDocumentInvalid( + f"{folder.name}: cannot read {_SCENARIO_JSON}: {exc}" + ) from exc + try: + body = json.loads(raw) + except json.JSONDecodeError as exc: + raise ScenarioDocumentInvalid( + f"{folder.name}: {_SCENARIO_JSON} is not valid JSON: {exc}" + ) from exc + if not isinstance(body, dict): + raise ScenarioDocumentInvalid(f"{folder.name}: {_SCENARIO_JSON} is not a JSON object") + + scenario_key = body.get("scenario_key", "") + if not isinstance(scenario_key, str): + raise ScenarioDocumentInvalid(f"{folder.name}: scenario_key is not a string") + scenario_id = body.get("scenario_id", "") + if not isinstance(scenario_id, str): + raise ScenarioDocumentInvalid(f"{folder.name}: scenario_id is not a string") + sub_goal_names = body.get("sub_goals", []) + if not isinstance(sub_goal_names, list) or not all( + isinstance(name, str) for name in sub_goal_names + ): + raise ScenarioDocumentInvalid(f"{folder.name}: sub_goals is not a list of strings") + for name in sub_goal_names: + _validate_subgoal_name(name, folder_name=folder.name) + + setup_code = _read_text(folder / _SETUP_PY, label=folder.name) + ready_code = _read_text(folder / _READY_PY, label=folder.name) + setup = _compile_entry(setup_code, label=f"{folder.name}/{_SETUP_PY}", entry="setup") + ready = _compile_entry(ready_code, label=f"{folder.name}/{_READY_PY}", entry="ready") + + sub_goals: list[_CompiledSubGoal] = [] + for name in sub_goal_names: + check_path = folder / _CHECKS_DIRNAME / f"{name}.py" + if check_path.is_file(): + check_code = _read_text(check_path, label=folder.name) + check = _compile_entry( + check_code, label=f"{folder.name}/{_CHECKS_DIRNAME}/{name}.py", entry="check", + allow_empty=False, # R1-2: an existing-but-empty check file is invalid, never a + # vacuous pass -- absence of the file is what means "judged". + ) + judged = "" + else: + # No `checks/.py` -- per `write_folder`'s own `deterministic()` filter, this + # name is a JUDGED sub-goal. + judged = _JUDGED_MARKER + check = _judged_placeholder_check + sub_goals.append(_CompiledSubGoal(name=name, judged=judged, check=check)) + + return _CompiledScenario( + scenario_key=scenario_key, + scenario_id=scenario_id, + sub_goals=tuple(sub_goals), + setup=setup, + ready=ready, + ) + + +def load_scenarios(bundle_dir: Path) -> list[_CompiledScenario]: + """Every scenario document under `/scenarios/`, compiled and wrapped, in the same + sorted-by-folder-name order `folder.py`'s `read_all` uses. Raises `ScenarioDocumentInvalid` on + the FIRST unreadable or malformed folder -- unlike `read_all`, which skips one and continues; + a hosted job has nobody watching a suite by hand to notice a scenario silently missing from the + count, so a folder this reader cannot use fails the whole job instead of shrinking it quietly. + """ + root = bundle_dir / SCENARIOS_DIRNAME + if not root.is_dir(): + raise ScenarioDocumentInvalid(f"{root} is not a directory") + try: + entries = sorted(root.iterdir()) + except OSError as exc: + # An unreadable `scenarios/` directory is the same typed failure as any other malformed + # document (R1-1) -- this is inside `run_job`'s `try`/`except ScenarioDocumentInvalid` + # (unlike `bundle_has_scenarios`'s own guard above), so raising here is the safe direction. + raise ScenarioDocumentInvalid(f"{root}: cannot list scenario folders: {exc}") from exc + scenarios: list[_CompiledScenario] = [] + for folder in entries: + if not folder.is_dir(): + continue + scenarios.append(_load_one(folder)) + if not scenarios: + raise ScenarioDocumentInvalid(f"{root} contains no scenario folders") + return scenarios + + +class BundleScenarioSource: + """The real `ScenarioSource`: reads and compiles the bundle's own scenario documents. + `hosted_entrypoint.run_job` wires this in only when the injected source is still the default + `NotWiredScenarioSource` AND the bundle actually carries a `scenarios/` directory (the LAYOUT + DECISION's presence test) -- an injected `ScenarioSource` (every test, every future caller) + always wins over this one. + """ + + async def build( + self, + job: Any, + bundle: Any, + scenarios_client: "ScenariosClient", + *, + pool: Any, + world_factory: Any, + bundle_dir: Path, + ) -> Sequence[_CompiledScenario]: + del bundle, pool, world_factory + # `Path.read_text`/`iterdir`/`compile` are all blocking filesystem+CPU work -- run off the + # event loop the same way `hosted_entrypoint.py` already does for `bundle_source.load` and + # `preflight_bundle`, rather than stalling every other in-flight scenario behind it. + try: + scenarios = await asyncio.wait_for( + asyncio.to_thread(load_scenarios, bundle_dir), timeout=_LOAD_TIMEOUT_SECONDS + ) + except asyncio.TimeoutError as exc: + # R1-5: the underlying thread cannot actually be canceled/killed -- it is left running + # in the background. Converting the hang into a typed terminal here is still strictly + # better than the pre-fix behavior (no terminal event, ever): the job gets an honest, + # bounded FAILED verdict instead of hanging until the platform's own wall clock gives up. + raise ScenarioDocumentInvalid( + f"{bundle_dir / SCENARIOS_DIRNAME}: loading scenario documents exceeded " + f"{_LOAD_TIMEOUT_SECONDS:.0f}s" + ) from exc + # An empty `scenario_key` is carried VERBATIM off the document by design (module docstring + # -- never synthesized here), but it is also the one shape `hosted_entrypoint.py`'s own + # downstream `_validate_scenarios` would reject as a local, deterministic ENVIRONMENT-domain + # content defect -- checked HERE, before `register_with_platform` ever reaches the network, + # so that cheaper, existing local classification wins over a round trip that would only + # rediscover the same defect as a `platform_sync` failure instead (Azain's serializer + # rejects a blank `scenario_key` with its own 400 -- `scenario_key` is a plain + # non-`allow_blank` `CharField`, hosted_harness.py:169). `ScenarioDocumentInvalid` reuses + # `run_job`'s EXISTING `except ScenarioDocumentInvalid` clause (domain=environment) -- + # nothing new to catch there. + if any(not scenario.scenario_key for scenario in scenarios): + raise ScenarioDocumentInvalid( + f"{bundle_dir / SCENARIOS_DIRNAME}: a scenario document has no non-empty " + "scenario_key" + ) + # p13: pre-allocation, after load and before the scheduler ever sees a scenario (spine + # step 3.5) -- `register_with_platform` raises `ScenarioPreallocationError`/ + # `ob.HostedFencedError`/`ob.HostedChannelFailedError`/`ob.HostedAttemptSupersededError` on + # any failure, all of which `hosted_entrypoint.run_job`'s existing call site around + # `scenario_source.build()` already maps to the typed `validating_scenarios`/`platform_sync` + # terminal (or the fenced exit) -- nothing new to catch here. + return await register_with_platform(scenarios_client, scenarios, run_name=job.run_id) + + +def _preallocation_error(code: str, message: str) -> Exception: + """Builds a `hosted_entrypoint.ScenarioPreallocationError` for a guard failure below -- + imported lazily (not at module level) because `hosted_entrypoint.py` imports THIS module at + its own top level (`BundleScenarioSource`/`ScenarioDocumentInvalid`/`bundle_has_scenarios`), so + a top-level import back would be a circular import. Reusing that exact exception class (rather + than inventing a new one) is what lets these guard failures land on `run_job`'s ALREADY-WIRED + `except (ScenarioSourceNotWired, ScenarioPreallocationError)` clause with no changes there. + """ + from .hosted_entrypoint import ScenarioPreallocationError + + return ScenarioPreallocationError( + ob.ChannelError(ob.ChannelOutcome.PERMANENT_ITEM, FailureDomain.PLATFORM_SYNC, code, message) + ) + + +def _provision_payload(run_name: str, scenarios: Sequence[_CompiledScenario]) -> dict[str, Any]: + """`HarnessScenarioProvisionSerializer`/`HarnessProvisionPersonaSerializer` + (futureagi/simulate/serializers/hosted_harness.py:168-190): `operation`/`name`/`personas` (with + each persona's `scenario_key`) are the only fields this module can actually supply -- `name`/ + `role`/`situation`/`outcome`/`persona` are all `required=False` on the real serializer, and + `_CompiledScenario` itself carries none of them BY DESIGN (this module's own LAYOUT DECISION, + see the module docstring: it reads `scenario.json` for the scheduler-facing fields only, never + through pr63's full `Scenario` model). Sending bare `scenario_key` per persona still validates + against the real endpoint; see CONTRACT NOTES in reports/p13-worker-r2.md. + """ + return { + "operation": "provision", + "name": run_name, + "personas": [{"scenario_key": scenario.scenario_key} for scenario in scenarios], + } + + +def _begin_payload(run_test_id: str, scenarios: Sequence[_CompiledScenario]) -> dict[str, Any]: + """`HarnessScenarioBeginSerializer` (futureagi/simulate/serializers/hosted_harness.py:193-198): + `scenario_keys` is `allow_empty=False` and REQUIRED, and `begin_scenarios` + (services/hosted_harness.py:323-329) 409s (`scenario_key_mismatch`) on anything but an EXACT + match against the full sealed set -- there is no "subset to run" semantics on the real + platform (Karthik's contract text describes an optional partial-subset `scenario_ids`; the + live route does not implement that -- CONTRACT NOTES). The full set is sent every time. + """ + return { + "operation": "begin", + "run_test_id": run_test_id, + "scenario_keys": [scenario.scenario_key for scenario in scenarios], + } + + +def _scenario_ids_by_key( + submitted: Sequence[_CompiledScenario], raw_scenarios: Any +) -> dict[str, str]: + """Matches the platform's KEYED provision response + (`{"scenarios": [{"scenario_key", "scenario_id"}, ...]}`, + futureagi/simulate/serializers/hosted_harness.py:251-260 + + services/hosted_harness.py:487-501's `_provision_response`) back onto `submitted` BY + `scenario_key` -- a dict lookup, never a positional zip. A positional zip (matching Karthik's + documented `scenario_ids` array shape, not what the platform actually returns) would silently + mismatch scenario_id -> scenario the instant the response order differs from `submitted`'s + order, which nothing on the wire guarantees. Every check below raises rather than returning a + partial mapping -- "never partial assignment" per the brief: the caller only gets a mapping + once it is proven complete (every submitted key present, exactly once) and exact (no + unrecognized key). + """ + if not isinstance(raw_scenarios, list): + raise _preallocation_error( + "scenarios_provision_response_invalid", "response 'scenarios' is not a list" + ) + by_key: dict[str, str] = {} + for entry in raw_scenarios: + if not isinstance(entry, dict): + raise _preallocation_error( + "scenarios_provision_response_invalid", "a 'scenarios' entry is not an object" + ) + key = entry.get("scenario_key") + scenario_id = entry.get("scenario_id") + if not isinstance(key, str) or not key: + raise _preallocation_error( + "scenarios_provision_response_invalid", + "a 'scenarios' entry has no non-empty scenario_key", + ) + if key in by_key: + raise _preallocation_error( + "scenario_registration_duplicate_key", + f"scenario_key {key!r} appears more than once in the provision response", + ) + if not isinstance(scenario_id, str) or not scenario_id: + raise _preallocation_error( + "scenarios_provision_response_invalid", + f"scenario_key {key!r} has no non-empty scenario_id", + ) + by_key[key] = scenario_id + + submitted_keys = [scenario.scenario_key for scenario in submitted] + unknown = sorted(set(by_key) - set(submitted_keys)) + if unknown: + raise _preallocation_error( + "scenario_registration_unknown_key", + f"provision response named scenario_key(s) never submitted: {unknown}", + ) + missing = sorted(set(submitted_keys) - set(by_key)) + if missing: + raise _preallocation_error( + "scenario_registration_missing", + f"provision response is missing scenario_key(s): {missing}", + ) + return by_key + + +async def register_with_platform( + scenarios_client: "ScenariosClient", + scenarios: Sequence[_CompiledScenario], + *, + run_name: str, +) -> Sequence[_CompiledScenario]: + """The scenario pre-allocation SEAM, now wired against the platform's real route (a single + `POST .../scenarios/`, discriminated by a body-level `operation` field -- see + `ScenariosClient`'s own docstring for the file:line evidence). `.provision()`/`.begin()` are + blocking network calls (same `ScenariosClient` the rest of `hosted_entrypoint.py` already + drives off the event loop via `asyncio.to_thread` -- matched here rather than diverging). + + Sequence: provision (get platform-assigned ids, keyed by `scenario_key`) -> match ids back + onto `scenarios` with hard guards (`_scenario_ids_by_key`, raises before ANY assignment on any + mismatch) -> begin (seals execution against the FULL scenario_keys set; a begin failure means + NO scenario in this batch is returned with an id -- the whole call raises, same as a provision + failure) -> only then build and return the new scenario list with `scenario_id` filled in. + """ + provision_result = await asyncio.to_thread( + scenarios_client.provision, _provision_payload(run_name, scenarios) + ) + run_test_id = provision_result.get("run_test_id") + if not isinstance(run_test_id, str) or not run_test_id: + raise _preallocation_error( + "scenarios_provision_response_invalid", "provision response has no run_test_id" + ) + id_by_key = _scenario_ids_by_key(scenarios, provision_result.get("scenarios")) + + await asyncio.to_thread(scenarios_client.begin, _begin_payload(run_test_id, scenarios)) + + return tuple( + replace(scenario, scenario_id=id_by_key[scenario.scenario_key]) for scenario in scenarios + ) diff --git a/src/fi/alk/harness/world/errors.py b/src/fi/alk/harness/world/errors.py new file mode 100644 index 00000000..c8fafc47 --- /dev/null +++ b/src/fi/alk/harness/world/errors.py @@ -0,0 +1,78 @@ +"""What the hosted world handle raises when scenario code asks it for something it will not do. + +Every one of these is scenario code at fault, never the world's contents and never the agent +under test: a `KeyError` from a missing table reads as a finding about data, and a bare +`StoreError` from the postgres driver reads as an infrastructure fault. Neither is right for +"you called `change` without saying which column `key` names," so the handle has its own +vocabulary, and folds it under one base class for whoever has to route "scenario code misused +the handle" to one outcome without naming all six. +""" + +from __future__ import annotations + + +class WorldError(RuntimeError): + """Scenario code asked the world handle for something it will not do.""" + + +class WorldReadOnly(WorldError): + """`put`, `change`, `drop` or `call` reached the handle `ready()` or `check()` were given. + + Those two only ever observe a run. A check that could write would be able to change the very + thing it is grading, and nothing downstream could tell the difference between a check that + found a problem and one that quietly fixed it. + """ + + +class WorldReservedName(WorldError): + """Scenario code named the harness's own conformance canary. + + That table exists to prove worlds are really isolated from each other, not to hold scenario + data, and it never appears in `state()` either. + """ + + +class WorldQueryRejected(WorldError): + """`query()` was handed something that is not one plain read. + + The database's own read-only transaction is what actually stops a write; this is the + friendlier rejection in front of it, so a statement that was never going to be allowed fails + on a message naming the reason rather than a lock error three layers down. + """ + + +class WorldStateTooLarge(WorldError): + """`state()` reached a table whose row count, measured when the baseline was frozen, passed + the cap. + + Measured once, at freeze time, by the provisioner — never recomputed here, so which tables + raise is fixed before a scenario ever runs and nothing a call does during one can move it. + """ + + +class WorldUnavailable(WorldError): + """The handle cannot do this, given how the world in front of it is built — not what it holds. + + A postgres world whose `public` schema has no tables, and `call()` — which raises + unconditionally until the `http_tool` shim's wire format is pinned somewhere in the contracts + — are both this: nothing went wrong, the capability was never there. + """ + + +class WorldUsageError(WorldError): + """A `put`, `change` or `drop` cannot be carried out as asked. + + Inserting into something that is not a table, or changing or dropping a record without + saying which column `key` names — hosted worlds cannot invent tables or guess a column, so + both are reported here rather than attempted. + """ + + +class WorldStoreless(WorldError): + """The bundle declared no SQL store, so this world holds no state at all. + + A stateless agent's scenarios can still run and pass — but the moment scenario code asks the + handle for state (`state`, `put`, `query`, ...), there is nothing behind the handle to answer + with. Raised eagerly, by name, so the receipt says "this scenario's checks need a store the + bundle never declared" instead of crashing three layers down in a driver. + """ diff --git a/src/fi/alk/harness/world/handle.py b/src/fi/alk/harness/world/handle.py new file mode 100644 index 00000000..b5e2699c --- /dev/null +++ b/src/fi/alk/harness/world/handle.py @@ -0,0 +1,521 @@ +"""The hosted world handle: the shipped world vocabulary, backed by one world's own postgres. + +`GeneratedWorld` is a database an agent's own generated handlers reach through `Db`. A hosted +world has no handlers to generate — the tables are whatever the agent's own migrations made, and +what a scenario needs is the same six-verb surface (`state`, `put`, `change`, `drop`, `call`, +`query`) built directly on the store, with nothing to adopt or reimplement per agent. Everything +this module refuses, it refuses before the database sees it, so a scenario's mistake reads as a +message naming what it did wrong rather than a `KeyError` or a driver traceback three layers down. +""" + +from __future__ import annotations + +import random +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from .errors import ( + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from .runtime import Call +from .stores.postgres import PostgresStore + +# The harness's own isolation canary. It exists to prove worlds are really separate from each +# other, never to hold scenario data, so scenario code is never allowed to see or touch it. +CONFORMANCE_TABLE = "_alk_conformance" + +# Measured once, at baseline freeze, by the provisioner — never recomputed here. A table over +# this stays over it for the whole run; nothing a scenario does can move which tables raise. +STATE_ROW_CAP = 5000 + +# The only statement shapes `query()` accepts. Anything else is refused before it reaches the +# database's own read-only transaction, so a statement that was never going to be allowed fails +# on a message naming why rather than a lock error from three layers down. +_READ_KEYWORDS = {"select", "with", "values"} + +# The read-only view's fallback answer is restricted to exactly this vocabulary, so a genuinely +# unknown attribute — a capability probe, a dunder, a typo — still reads as a plain +# `AttributeError` instead of masquerading as a write refusal. +_WRITE_VERBS = frozenset({"put", "change", "drop", "call"}) + + +class HostedWorld: + """The `World` surface, backed by this scenario's own logical postgres database. + + One handle per scenario, over one short-lived autocommit connection per operation — nothing + held open, which is what lets `reset` drop the database out from under a discarded world. + `world_index` and `rng` are plain data: the index is for diagnostics only, and the generator + is the only sanctioned source of randomness scenario code may use. + """ + + def __init__( + self, + store: PostgresStore, + world_index: int, + rng: random.Random, + baseline_row_counts: Mapping[str, int], + ) -> None: + """Wrap `store` as the `World` surface for one scenario. + + `baseline_row_counts` is keyed by the bare `pg_tables.tablename` value — no schema + prefix, no quoting — for every table `public` held when the baseline was frozen. A + visible table missing from this map has no measured cap to enforce, so the coverage + check below fails construction outright rather than waiting for a scenario's first + access to discover a provisioning gap through its own retry. + """ + self._store = store + self.world_index = world_index + self.rng = rng + # Row counts as they stood when the baseline was frozen, keyed by table. Never + # re-measured: a live count would make the cap depend on what a scenario already wrote, + # and the whole point is that it is decided before any scenario runs. + self._baseline_row_counts = dict(baseline_row_counts) + self._require_baseline_coverage(self._visible_tables()) + + # -- reading ------------------------------------------------------------------------------ + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + """A snapshot of the public schema, or of one table in it. + + Bare `state()` leaves an over-cap table out of the snapshot rather than raising through + it — one seeded audit table must not make the primary read verb inert for the whole + run. A table nothing measured at baseline freeze gets the same treatment: the only way + one exists is a table the agent under test created since construction, and nothing it + does during a call may decide whether bare `state()` raises. Naming either kind of table + explicitly (`state("big_table")`) still raises: the exclusion is a property of the + snapshot, not a way to read the table around its own cap. If every visible table is + over-cap or unmeasured, the exclusion would leave the snapshot `{}` — the one thing + state() must never return, since an empty snapshot reads as an observation and makes a + negative check pass on a world nobody actually looked at — so that case raises too, + naming the tables it would have excluded. The exclusion itself happens at the read: the + store is asked for only the included tables, not for every table with the excluded ones + thrown away afterward, so one huge seeded table can no longer make every bare `state()` + pay to materialise and discard rows nobody asked to see. + """ + self._reject_reserved(table) + if table is not None: + visible = self._visible_tables() + self._require_nonempty_schema(visible) + if table not in visible: + raise WorldUsageError( + f"{table!r} is not a table in this world; it holds {sorted(visible)}." + ) + count = self._row_count(table) + if count > STATE_ROW_CAP: + raise WorldStateTooLarge( + f"{table!r} held {count} rows when the baseline " + f"was frozen, over the {STATE_ROW_CAP}-row cap; state() will not read it " + "back." + ) + return {table: self._store.table(table)} + + names = self._visible_tables() + self._require_nonempty_schema(names) + # A table nothing measured at freeze is treated the same as an over-cap one here, not + # routed through `_row_count`'s typed refusal — that refusal is for a scenario naming a + # table explicitly; the bare snapshot must not go unavailable over a table the agent + # itself created since construction (the only actor besides the provisioner that can). + included = [ + name + for name in names + if name in self._baseline_row_counts + and self._baseline_row_counts[name] <= STATE_ROW_CAP + ] + if not included: + raise WorldStateTooLarge( + f"every table this world holds — {sorted(names)} — is over the " + f"{STATE_ROW_CAP}-row cap or was never measured at baseline freeze; state() " + "will not return {} in their place." + ) + # One connection, asked for only the included tables — a bare state() used to open a + # fresh connection per table (and a second one just to look up its primary key) to read + # every table including the over-cap ones, then throw the over-cap rows away here. + return self._store.state(only=included) + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + """The read escape hatch: one statement, on a transaction that cannot write. + + The database's own read-only transaction is the actual guard; this token check only + makes the common mistake — a stray write, a second statement — fail with a reason + attached instead of a lock error from underneath. + """ + _reject_unless_read(sql) + return self._store.query(sql, tuple(params)) + + # -- writing ------------------------------------------------------------------------------ + + def put(self, collection: str, record: Mapping[str, Any], *, key: str = "") -> dict[str, Any]: + """Insert one record; return exactly what the table stored, generated key included. + + `key` exists only to keep this signature a superset of `GeneratedWorld.put`; a hosted + table already knows its own key — the column its own migrations gave it — so a + scenario naming one here would be telling the table what to call a value the table is + about to generate itself. + """ + self._reject_reserved(collection) + if key: + raise WorldUsageError("a hosted table's key is the table's own — do not pass one.") + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so put() only reaches what the agent's own migrations made." + ) + return self._store.add(collection, dict(record)) + + def change( + self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" + ) -> int: + """Update matching records; return how many changed.""" + self._reject_reserved(collection) + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so change() only reaches what the agent's own migrations made." + ) + by = by or self._resolve_by(collection) + if by not in self._table_columns(collection): + raise WorldUsageError( + f"change({collection!r}, {key!r}, ...) was given by={by!r}, which is not a " + f"column of {collection!r}." + ) + return self._store.amend(collection, key, dict(changes), by=by) + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + """Delete matching records, or every record when `key` is empty; return the count.""" + self._reject_reserved(collection) + if collection not in self._visible_tables(): + raise WorldUsageError( + f"{collection!r} is not a table in this world; hosted worlds cannot invent " + "one, so drop() only reaches what the agent's own migrations made." + ) + if key: + by = by or self._resolve_by(collection) + if by not in self._table_columns(collection): + raise WorldUsageError( + f"drop({collection!r}, {key!r}) was given by={by!r}, which is not a " + f"column of {collection!r}." + ) + return self._store.remove(collection, key, by=by) + + def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: + """Play one of the agent's own tools against this world. + + Not implemented: the `http_tool` evidence seam's wire format is not pinned anywhere in + the contracts yet, and guessing at a shape here would ship one nobody agreed to and + scenario code would end up depending on. Raising unconditionally is the honest answer + until the evidence layer pins it — run-time scenario code should not need this anyway, + since `setup` already runs at proof time and the data verbs cover the rest. + """ + raise WorldUnavailable( + "the http_tool shim wire format is not yet pinned by the contracts — report, " + "don't guess." + ) + + def read_only(self) -> "ReadOnlyWorld": + """The view `ready` and `check` run against: every write verb refuses outright. + + A check able to write could not be told apart from one that quietly repaired what it was + supposed to be grading, so this is what stands between those two functions and the real + handle. + """ + return ReadOnlyWorld(self) + + # -- internal ----------------------------------------------------------------------------- + + def _reject_reserved(self, collection: str | None) -> None: + if collection == CONFORMANCE_TABLE: + raise WorldReservedName( + f"{collection!r} is the harness's own conformance canary, not scenario data; " + "it never appears to scenario code." + ) + + def _require_nonempty_schema(self, visible: list[str]) -> None: + if not visible: + raise WorldUnavailable( + "this world's public schema holds no tables to observe; a postgres world with " + "nothing in it is not a world state() can honestly report on." + ) + + def _require_baseline_coverage(self, names: list[str]) -> None: + """Refuse rather than assume a table the baseline never measured is under the cap. + + A missing entry used to default to a row count of 0, which would let a table nobody + measured at freeze read back as though it were known to be small; failing loud here is + the whole point of deciding the cap before any scenario runs instead of guessing at it. + Called once, from `__init__`: the table set and the baseline are both fixed for the + life of the handle, so checking again on every later access would only repeat a answer + construction already gave. + """ + missing = [name for name in names if name not in self._baseline_row_counts] + if missing: + raise WorldUnavailable( + f"the baseline row counts this world was built with never measured " + f"{sorted(missing)}; state()'s cap cannot be decided for a table nobody " + "measured at freeze." + ) + + def _row_count(self, table: str) -> int: + """This table's baseline row count, or the same typed refusal `_require_baseline_coverage` + gives a construction-time gap — for a table that only shows up afterward, so it never + reaches the caller as a bare `KeyError`. + """ + try: + return self._baseline_row_counts[table] + except KeyError: + raise WorldUnavailable( + f"the baseline row counts this world was built with never measured " + f"{table!r}; state()'s cap cannot be decided for a table nobody measured at " + "freeze." + ) from None + + def _visible_tables(self) -> list[str]: + rows = self._store.query( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename" + ) + return [row["tablename"] for row in rows if row["tablename"] != CONFORMANCE_TABLE] + + def _table_columns(self, table: str) -> set[str]: + rows = self._store.query( + "SELECT column_name FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s", + (table,), + ) + return {row["column_name"] for row in rows} + + def _primary_key_order(self, table: str) -> list[str]: + """This table's primary key columns, in index order. + + Joined through `pg_class`/`pg_namespace` rather than a `%s::regclass` cast over an + f-string, so a table name is only ever a bound value — an embedded `"` (a table created + as `CREATE TABLE "we""ird" (...)`) is just a character in that value instead of + something a regclass cast has to parse. + """ + rows = self._store.query( + """ + SELECT a.attname + FROM pg_index i + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = %s AND i.indisprimary + ORDER BY array_position(i.indkey, a.attnum) + """, + (table,), + ) + return [row["attname"] for row in rows] + + def _resolve_by(self, table: str) -> str: + """The column to key `change`/`drop` on when scenario code did not name one. + + A single-column primary key is the only case unambiguous enough to guess; no primary + key at all, or a composite one, means the store genuinely cannot tell which column + `key` names, so the scenario has to say. + """ + columns = self._primary_key_order(table) + if len(columns) == 1: + return columns[0] + raise WorldUsageError( + f"change/drop on {table!r} needs by=; it has no single-column primary " + "key to default to." + ) + + +class ReadOnlyWorld: + """A `World` whose write verbs are refused before they reach `HostedWorld` at all.""" + + def __init__(self, world: HostedWorld) -> None: + # Name-mangled so code outside this class reaching for `._world` cannot casually + # recover the writable handle a "read-only" view exists to stand in front of. + self.__world = world + self.world_index = world.world_index + self.rng = world.rng + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return self.__world.state(table) + + def query(self, sql: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + return self.__world.query(sql, params) + + def put(self, collection: str, record: Mapping[str, Any], *, key: str = "") -> dict[str, Any]: + raise WorldReadOnly( + f"put({collection!r}, ...) reached a read-only handle; ready() and check() only " + "ever observe a run." + ) + + def change( + self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" + ) -> int: + raise WorldReadOnly( + f"change({collection!r}, {key!r}, ...) reached a read-only handle; ready() and " + "check() only ever observe a run." + ) + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + raise WorldReadOnly( + f"drop({collection!r}, {key!r}) reached a read-only handle; ready() and check() " + "only ever observe a run." + ) + + def call(self, name: str, arguments: Mapping[str, Any] | None = None) -> Call: + raise WorldReadOnly( + f"call({name!r}, ...) reached a read-only handle; ready() and check() only ever " + "observe a run." + ) + + def __getattr__(self, name: str) -> Any: + """A write verb `HostedWorld` grows later, with no override here yet, still reads as + `WorldReadOnly` rather than a plain `AttributeError` indistinguishable from a typo. + + Restricted to that known vocabulary and never to dunders: this repo's own runner code + reaches for `hasattr(world, "forward")` and `getattr(world, "runtime_tools", set())` on + world objects, and both only work through the ordinary `AttributeError` those tools + expect from a name that is simply not there, not from a refusal that happens to look + like one. + """ + if name.startswith("__") and name.endswith("__"): + raise AttributeError(name) + if name in _WRITE_VERBS: + raise WorldReadOnly( + f"{name!r} reached a read-only handle; ready() and check() only ever observe " + "a run." + ) + raise AttributeError(name) + + +def _is_word_char(char: str) -> bool: + return char.isalnum() or char == "_" + + +_DOLLAR_TAG = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)?\$") + + +def _dollar_quote_end(sql: str, start: int) -> int | None: + """The index just past the matching closing `$tag$`, if `sql[start:]` opens one. + + `None` if `start` is not a dollar-quote opener at all — a `$1` parameter placeholder, or a + bare `$`, never matches the tag grammar and is left for the caller to treat as an ordinary + character. An opener with no matching close consumes to the end of the string, the same + fate an unterminated `'`/`"` span already gets below. + """ + opener = _DOLLAR_TAG.match(sql, start) + if opener is None: + return None + delimiter = opener.group(0) + end = sql.find(delimiter, opener.end()) + return len(sql) if end == -1 else end + len(delimiter) + + +def _blank(sql: str, quote_chars: str) -> str: + """`sql` with every comment blanked, plus any quoted span opened by a character in + `quote_chars`, plus every dollar-quoted `$tag$...$tag$` span. + + Two callers need two different blindnesses: the shape check does not care what a string + literal's characters are, so it blanks both quote styles; the reserved-name check must not + let a quoted identifier hide inside a span it no longer looks at, so it blanks only string + literals and leaves double-quoted identifiers as text. Dollar-quoting is blanked for both + callers regardless of `quote_chars` — it is never an identifier, only ever a literal, and a + `'` or `;` sitting inside one is exactly what both callers must not see as SQL. + + A `'` immediately after a bare `E`/`e` opens an escape string, where a `\\` escapes whatever + follows it the same way doubling the quote does. Missing that let a `\\'` inside one close + the literal early: the real closing quote right after it then read as opening a fresh span, + and everything up to the next quote — semicolon, second statement and all — vanished into + it as though it were still part of the string. + """ + out: list[str] = [] + i, n = 0, len(sql) + while i < n: + if sql.startswith("--", i): + end = sql.find("\n", i) + i = n if end == -1 else end + 1 + continue + if sql.startswith("/*", i): + end = sql.find("*/", i + 2) + i = n if end == -1 else end + 2 + continue + char = sql[i] + if char == "$": + dollar_end = _dollar_quote_end(sql, i) + if dollar_end is not None: + i = dollar_end + out.append(" ") + continue + if char in quote_chars: + escapes = ( + char == "'" + and i > 0 + and sql[i - 1] in "Ee" + and (i == 1 or not _is_word_char(sql[i - 2])) + ) + i += 1 + while i < n: + if escapes and sql[i] == "\\": + i += 2 + continue + if sql[i] == char: + if sql[i : i + 2] == char * 2: # an escaped quote inside the literal + i += 2 + continue + i += 1 + break + i += 1 + out.append(" ") + continue + out.append(char) + i += 1 + return "".join(out) + + +def _sql_skeleton(sql: str) -> str: + """`sql` with every comment and string literal blanked out to a single space. + + The token check only needs to know what kind of statement this is and how many of them + there are; without this a semicolon or the word FOR inside a quoted value would count as + SQL, and the check would end up rejecting a value instead of a statement. + """ + return _blank(sql, "'\"") + + +def _names_the_reserved_table(sql: str) -> bool: + """Whether `_alk_conformance` appears anywhere as an identifier, quoted or not. + + Blanked for string literals only, deliberately not double-quoted identifiers — an + identifier is exactly where this name could hide from a check that blanked those away too. + The boundary either side of the name is `\\w`, not a quote: a `"` sitting right against it + is exactly the character that must not shield it, or `"_alk_conformance"` would read as + hidden the same way a comment or a literal already is. + """ + identifiers = _blank(sql, "'") + pattern = rf"(? None: + body = _sql_skeleton(sql).strip() + if not body: + raise WorldQueryRejected("query() was given nothing to run.") + unterminated = body[:-1].strip() if body.endswith(";") else body + if ";" in unterminated: + raise WorldQueryRejected("query() runs one statement; this text holds more than one.") + leading = re.match(r"[A-Za-z_]+", unterminated) + word = leading.group(0).lower() if leading else "" + if word not in _READ_KEYWORDS: + raise WorldQueryRejected( + f"query() only reads: it takes SELECT, WITH or VALUES, not {word or sql[:20]!r}." + ) + if re.search(r"\bfor\s+(update|share)\b", unterminated, re.IGNORECASE): + raise WorldQueryRejected( + "query() runs on a read-only transaction; FOR UPDATE/FOR SHARE lock rows for a " + "write that can never follow." + ) + if _names_the_reserved_table(sql): + raise WorldQueryRejected( + f"query() refuses to name {CONFORMANCE_TABLE!r}; it is the harness's own " + "conformance canary, not scenario data." + ) diff --git a/src/fi/alk/harness/world/runtime.py b/src/fi/alk/harness/world/runtime.py index e60b1e1a..9deb5ac9 100644 --- a/src/fi/alk/harness/world/runtime.py +++ b/src/fi/alk/harness/world/runtime.py @@ -83,7 +83,7 @@ def find(self, collection: str, **fields: Any) -> list[dict[str, Any]]: if all(record.get(field) == value for field, value in fields.items()) ] - def add(self, collection: str, record: Mapping[str, Any]) -> int: + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: return self.store.add(collection, record) diff --git a/src/fi/alk/harness/world/stores/__init__.py b/src/fi/alk/harness/world/stores/__init__.py index 438ace01..4f77002a 100644 --- a/src/fi/alk/harness/world/stores/__init__.py +++ b/src/fi/alk/harness/world/stores/__init__.py @@ -85,7 +85,8 @@ def collections(self) -> list[str]: ... def holds(self, collection: str) -> bool: ... def records(self, collection: str) -> list[dict[str, Any]]: ... def state(self) -> dict[str, list[dict[str, Any]]]: ... - def add(self, collection: str, record: Mapping[str, Any]) -> int: ... + def table(self, name: str) -> list[dict[str, Any]]: ... + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: ... def amend( self, collection: str, key: str, changes: Mapping[str, Any], *, by: str = "" ) -> int: ... @@ -154,7 +155,13 @@ def query(self, statement: str, params: Sequence[Any] = ()) -> list[dict[str, An "records() or state()." ) - def add(self, collection: str, record: Mapping[str, Any]) -> int: + def table(self, name: str) -> list[dict[str, Any]]: + raise StoreError( + f"{self.engine} does not read one table at a time. Read what it holds with " + "records() or state()." + ) + + def add(self, collection: str, record: Mapping[str, Any]) -> int | dict[str, Any]: raise StoreError(_UNWRITABLE.format(engine=self.engine, verb="add to")) def amend( diff --git a/src/fi/alk/harness/world/stores/container.py b/src/fi/alk/harness/world/stores/container.py index 7544ab58..3cbe08e1 100644 --- a/src/fi/alk/harness/world/stores/container.py +++ b/src/fi/alk/harness/world/stores/container.py @@ -169,8 +169,13 @@ def _await_ready(self) -> None: last = exc time.sleep(0.25) logs = docker("logs", "--tail", "20", self.container, check=False) + container = self.container + # A container that never answers is not a container to leave running -- `start()` already + # set `_started`, so without this the caller's own teardown never runs (nothing ever calls + # `stop()` on a store whose `start()` raised) and the container leaks for good. + self.stop() raise StoreError( - f"{self.container} did not answer within {READY_TIMEOUT_SECONDS:.0f}s: {last}\n" + f"{container} did not answer within {READY_TIMEOUT_SECONDS:.0f}s: {last}\n" f"last lines of its log:\n{logs}" ) diff --git a/src/fi/alk/harness/world/stores/postgres.py b/src/fi/alk/harness/world/stores/postgres.py index 0a89b444..71fbb131 100644 --- a/src/fi/alk/harness/world/stores/postgres.py +++ b/src/fi/alk/harness/world/stores/postgres.py @@ -22,6 +22,7 @@ from pathlib import Path from typing import Any +from ..errors import WorldQueryRejected from . import Held, Snapshot, StoreError from .container import ContainerStore, docker @@ -93,6 +94,34 @@ def execute(self, statement: str, params: Sequence[Any] = ()) -> int: cursor = connection.execute(statement, tuple(params)) return max(0, int(cursor.rowcount or 0)) + def query(self, statement: str, params: Sequence[Any] = ()) -> list[dict[str, Any]]: + """Run one read statement, on a connection Postgres itself will not let write. + + Autocommit, with the session's own default flipped to read-only rather than one shared + ``SET TRANSACTION READ ONLY`` transaction: every statement becomes its own implicit + read-only transaction, so nothing here is ever held open, and ``reset``'s drop of the + database can always proceed regardless of what a caller just read. + + An empty ``params`` tuple is passed through as ``None`` rather than as itself: psycopg + scans for placeholders whenever it is handed anything other than ``None``, and an + ordinary ``LIKE '%turkey%'`` with nothing to bind then reads its own ``%t`` as an + unmatched one and raises before the statement ever reaches Postgres. + """ + with _psycopg().connect( + self.dsn(), autocommit=True, options="-c default_transaction_read_only=on" + ) as connection: + cursor = connection.execute(statement, tuple(params) if params else None) + columns = [description[0] for description in cursor.description or []] + seen: set[str] = set() + for column in columns: + if column in seen: + raise WorldQueryRejected( + f"query() returned more than one column named {column!r}; alias one " + "of them so a row does not silently lose one under the other." + ) + seen.add(column) + return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + def _tables(self, connection: Any) -> list[str]: rows = connection.execute( "SELECT tablename FROM pg_tables WHERE schemaname = 'public' ORDER BY tablename" @@ -100,16 +129,24 @@ def _tables(self, connection: Any) -> list[str]: return [row[0] for row in rows] def _primary_key(self, connection: Any, table: str) -> list[str]: - """The primary key columns, used only to read rows back in a stable order.""" + """The primary key columns, used only to read rows back in a stable order. + + Joined through ``pg_class``/``pg_namespace`` rather than a ``%s::regclass`` cast over an + f-string, so a table name is only ever a bound value — an embedded ``"`` (a table + created as ``CREATE TABLE "we""ird" (...)``) is just a character in that value instead + of something a regclass cast has to parse. + """ rows = connection.execute( """ SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) - WHERE i.indrelid = %s::regclass AND i.indisprimary + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = %s AND i.indisprimary ORDER BY array_position(i.indkey, a.attnum) """, - (f'public."{table}"',), + (table,), ).fetchall() return [row[0] for row in rows] @@ -125,28 +162,47 @@ def _column_types(self, connection: Any, table: str) -> dict[str, str]: ).fetchall() return {row[0]: row[1] for row in rows} - def state(self) -> dict[str, list[dict[str, Any]]]: + def _select_ordered(self, connection: Any, table: str) -> list[dict[str, Any]]: + """Every row of one table, ordered by its primary key where it has one. + + Without that order the same data comes back in whatever sequence the heap happens to + hold it, and a check comparing the first row is reading a coin toss rather than the + agent's behaviour. Built with ``sql.Identifier`` rather than an f-string because + ``table`` is a name the harness only just read out of the catalogue, not a literal it + wrote itself. + """ + sql = _psycopg().sql + key = self._primary_key(connection, table) + statement = sql.SQL("SELECT * FROM {}").format(sql.Identifier(table)) + if key: + statement += sql.SQL(" ORDER BY ") + sql.SQL(", ").join( + sql.Identifier(column) for column in key + ) + cursor = connection.execute(statement) + columns = [description[0] for description in cursor.description or []] + return [dict(zip(columns, row, strict=True)) for row in cursor.fetchall()] + + def state(self, only: Sequence[str] | None = None) -> dict[str, list[dict[str, Any]]]: """Every table and its rows, in the shape the checks already expect. - Ordered by primary key where there is one. Without that the same data comes back in - whatever order the heap happens to hold it, and a check comparing the first row is - reading a coin toss rather than the agent's behaviour. + ``only`` narrows the read to the named tables, still inside the one connection — a + caller that already knows it wants a subset (``HostedWorld`` excluding over-cap tables) + never pays to read and discard rows for the ones it does not. ``None`` reads every table + in ``public``, exactly as before. """ with self._connect() as connection: - out: dict[str, list[dict[str, Any]]] = {} - for table in self._tables(connection): - key = self._primary_key(connection, table) - order = ( - " ORDER BY " + ", ".join(f'"{column}"' for column in key) - if key - else "" - ) - cursor = connection.execute(f'SELECT * FROM "{table}"{order}') - columns = [description[0] for description in cursor.description or []] - out[table] = [ - dict(zip(columns, row, strict=True)) for row in cursor.fetchall() - ] - return out + tables = self._tables(connection) if only is None else list(only) + return {table: self._select_ordered(connection, table) for table in tables} + + def table(self, name: str) -> list[dict[str, Any]]: + """One table's rows, ordered by primary key where it has one. + + The read-side counterpart to ``state()``'s per-table loop, for a caller that wants only + one of them: still a single connection, so reading one table never costs a second round + trip just to learn how to order it. + """ + with self._connect() as connection: + return self._select_ordered(connection, name) # -- how to put it back ---------------------------------------------------------- @@ -244,30 +300,57 @@ def load_from(self, path: str | Path) -> None: # -- what a scenario changes ----------------------------------------------------- - def add(self, collection: str, record: Any) -> int: + def add(self, collection: str, record: Any) -> dict[str, Any]: + """Insert one record and hand back exactly what Postgres stored. + + ``RETURNING *`` rather than a second read: a caller after the row's generated key (an + identity column, a default, a trigger) would otherwise have to guess which column that + is, and a table with no natural way to re-select the row it just inserted could not be + read back at all. + """ columns = list(record) - quoted = ", ".join(f'"{column}"' for column in columns) - placeholders = ", ".join(["%s"] * len(columns)) + sql = _psycopg().sql + statement = sql.SQL("INSERT INTO {} ({}) VALUES ({}) RETURNING *").format( + sql.Identifier(collection), + sql.SQL(", ").join(sql.Identifier(column) for column in columns), + sql.SQL(", ").join(sql.SQL("%s") for _ in columns), + ) with self._connect() as connection: types = self._column_types(connection, collection) cursor = connection.execute( - f'INSERT INTO "{collection}" ({quoted}) VALUES ({placeholders})', + statement, tuple( _adapt(record[column], types.get(column, "")) for column in columns ), ) - return cursor.rowcount + stored = cursor.fetchone() + if stored is None: + raise StoreError( + f"INSERT INTO {collection!r} ... RETURNING * came back with no row; a rule " + "or a BEFORE INSERT trigger the agent's own migrations declared can turn an " + "insert into a no-op, and put() cannot report a record that was never " + "written." + ) + out = [description[0] for description in cursor.description or []] + return dict(zip(out, stored, strict=True)) def amend(self, collection: str, key: str, changes: Any, *, by: str = "") -> int: if not by: raise StoreError( f"{collection} is a table, so changing a record needs the column it is keyed on" ) - sets = ", ".join(f'"{column}" = %s' for column in changes) + sql = _psycopg().sql + statement = sql.SQL("UPDATE {} SET {} WHERE {} = %s").format( + sql.Identifier(collection), + sql.SQL(", ").join( + sql.SQL("{} = %s").format(sql.Identifier(column)) for column in changes + ), + sql.Identifier(by), + ) with self._connect() as connection: types = self._column_types(connection, collection) cursor = connection.execute( - f'UPDATE "{collection}" SET {sets} WHERE "{by}" = %s', + statement, ( *( _adapt(value, types.get(column, "")) @@ -283,9 +366,10 @@ def remove(self, collection: str, key: str = "", *, by: str = "") -> int: raise StoreError( f"{collection} is a table, so removing one record needs the column it is keyed on" ) - statement = f'DELETE FROM "{collection}"' + ( - f' WHERE "{by}" = %s' if key else "" - ) + sql = _psycopg().sql + statement = sql.SQL("DELETE FROM {}").format(sql.Identifier(collection)) + if key: + statement += sql.SQL(" WHERE {} = %s").format(sql.Identifier(by)) with self._connect() as connection: cursor = connection.execute(statement, (key,) if key else ()) return cursor.rowcount diff --git a/src/fi/simulate/runtime/spec.py b/src/fi/simulate/runtime/spec.py index 681d19f0..15bab234 100644 --- a/src/fi/simulate/runtime/spec.py +++ b/src/fi/simulate/runtime/spec.py @@ -70,6 +70,9 @@ class RuntimeRequirements(BaseModel): concurrency_weight: int = Field(default=1, ge=1) max_duration_seconds: int = Field(default=300, ge=1) network_policy: str = "live" + # World count for the hosted harness (seam contract §1); the gateway caps + # it at admission, so the model stays permissive beyond ge=1. + parallelism: int = Field(default=1, ge=1) class TimeoutPolicy(BaseModel): diff --git a/tests/harness/test_bundle_v2.py b/tests/harness/test_bundle_v2.py new file mode 100644 index 00000000..c581dbf9 --- /dev/null +++ b/tests/harness/test_bundle_v2.py @@ -0,0 +1,807 @@ +"""`futureagi.environment-bundle.v2` model validation, per `hosted-execution-seams.md` v1.7 §2. + +Two lanes: the spec's own §2a/§2b/§2c example structures, transcribed here and proven to parse +(the model-layer accept side), against the rejections the model is responsible for on its own — +wrong schema version, an unknown process kind, a store with no sentinel, a capability whose +protocol disagrees with the engine actually backing it, a sentinel shape that disagrees with its +capability's protocol, a strategy the capability's protocol-implied engine does not support, a +store on a capability protocol this module cannot seed at all, unresolved +`service`/`control_service`/capability references (scoped to `kind: process`, per p3-round2's B3), +a duplicate process name, and a resolved secret value anywhere in the manifest. +`compute_inputs_digest` is checked against a hand-computed vector, not by calling back into itself. + +Rules that need a repo checkout, the job the bundle will run under, or the §2e checklist +(`compose_not_hosted`, `engine_unsupported`, `no_sql_store`, `depends_on` cycles, placeholder +vocabulary, reserved-name scanning) belong to Phase 4's preflight and are out of scope here. +""" + +from __future__ import annotations + +import hashlib +import json + +import pytest +from pydantic import ValidationError + +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + BundleRuntimeV2, + BundleV2Error, + EnvironmentBundleV2, + ManagedEngine, + ManagedProcess, + SourceProcess, + StoreEntry, + compute_inputs_digest, + load_bundle_v2, + seal_bundle_v2, +) + +# --- §2a/§2b/§2c: the spec's own examples, transcribed verbatim ----------------------------- + +RUNTIME_EXAMPLE = {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"} + +POSTGRES_PROCESS_EXAMPLE = { + "name": "postgres", + "kind": "managed", + "engine": "postgres", + "version": "16", + "user": "svc-data", + "depends_on": [], +} + +TOOLS_API_PROCESS_EXAMPLE = { + "name": "tools-api", + "kind": "source", + "working_directory": "services/tools-api", + "build_commands": [["npm", "ci"]], + "run_command": ["node", "server.js"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "PORT": "{{PORT_tools-api}}", + "TMPDIR": "{{WORLD_DIR}}", + }, + "secret_purposes": [], + "user": "svc-tools", + "depends_on": ["postgres"], +} + +AGENT_PROCESS_EXAMPLE = { + "name": "agent", + "kind": "source", + "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent/agent.py", "start"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "TOOLS_API_URL": "{{TOOLS_API_URL}}", + "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], + "user": "svc-agent", + "depends_on": ["postgres", "tools-api"], +} + +# §2b catalog-table engines beyond postgres, for the capability-protocol/sentinel/strategy pairing +# tests (F1) — a store's engine now comes from its capability's protocol, so these tests need a +# real managed process of the matching engine for the capability's `service` to resolve to. +REDIS_PROCESS_EXAMPLE = { + "name": "cache", + "kind": "managed", + "engine": "redis", + "version": "7", + "user": "svc-data", + "depends_on": [], +} + +RABBITMQ_PROCESS_EXAMPLE = { + "name": "queue", + "kind": "managed", + "engine": "rabbitmq", + "version": "3.13", + "user": "svc-data", + "depends_on": [], +} + +# The spec's own `<64-hex>` placeholder, filled with a real hex string — the example is only +# illustrating the digest's shape, not a value to reproduce. +SEED_STORE_EXAMPLE = { + "capability": "database", + "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + "baseline": { + "strategy": "template_database", + "inputs_digest": "sha256:" + "a" * 64, + }, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "12"}, +} + +FULL_MANIFEST_EXAMPLE = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "demo", + "runtime": RUNTIME_EXAMPLE, + "processes": [POSTGRES_PROCESS_EXAMPLE, TOOLS_API_PROCESS_EXAMPLE, AGENT_PROCESS_EXAMPLE], + "seed": {"stores": [SEED_STORE_EXAMPLE]}, + "capabilities": { + "database": { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "DATABASE_URL", + }, + "tools": { + "protocol": "http", + "service": "tools-api", + "configuration_name": "TOOLS_API_URL", + }, + }, + "readiness": [{"capability": "tools", "path": "/healthz"}], + "files": [{"path": "db/schema.sql", "sha256": "b" * 64, "size": 10}], + "provenance": { + "source_kind": "repository", + "repository": "org/repo", + "commit": "a" * 40, + # Bare 64-hex — what v1's `source_fingerprint` producer actually emits, unlike the + # `sha256:`-prefixed `digest`/`inputs_digest`. + "source_digest": "c" * 64, + }, +} + + +def test_the_runtime_example_from_2a_is_accepted() -> None: + runtime = BundleRuntimeV2.model_validate(RUNTIME_EXAMPLE) + assert runtime.kind.value == "process" + assert runtime.evidence_seam.value == "http_tool" + + +def test_the_managed_process_example_from_2b_is_accepted() -> None: + process = ManagedProcess.model_validate(POSTGRES_PROCESS_EXAMPLE) + assert process.engine.value == "postgres" + assert process.version == "16" + + +@pytest.mark.parametrize( + "example", [TOOLS_API_PROCESS_EXAMPLE, AGENT_PROCESS_EXAMPLE], ids=["tools-api", "agent"] +) +def test_the_source_process_examples_from_2b_are_accepted(example: dict) -> None: + process = SourceProcess.model_validate(example) + assert process.run_command + assert process.working_directory == example["working_directory"] + + +def test_the_seed_store_example_from_2c_is_accepted() -> None: + store = StoreEntry.model_validate(SEED_STORE_EXAMPLE) + assert store.baseline.strategy.value == "template_database" + assert store.sentinel.implied_engine is ManagedEngine.POSTGRES + + +def test_the_full_manifest_assembled_from_the_spec_examples_is_accepted() -> None: + bundle = EnvironmentBundleV2.model_validate(FULL_MANIFEST_EXAMPLE) + assert bundle.name == "demo" + assert [process.name for process in bundle.processes] == ["postgres", "tools-api", "agent"] + assert bundle.seed is not None and bundle.seed.stores[0].capability == "database" + + +# --- rejections the model is responsible for on its own ------------------------------------- + + +def test_wrong_schema_version_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "schema_version": "futureagi.environment-bundle.v1"} + with pytest.raises(ValidationError, match="bundle_schema_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_unknown_process_kind_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [{**POSTGRES_PROCESS_EXAMPLE, "kind": "container"}], + } + with pytest.raises(ValidationError, match="union_tag_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_with_no_sentinel_is_rejected() -> None: + incomplete = {key: value for key, value in SEED_STORE_EXAMPLE.items() if key != "sentinel"} + with pytest.raises(ValidationError, match="sentinel"): + StoreEntry.model_validate(incomplete) + + +# The engine a store answers to is now the engine behind its *capability's protocol* (F1), decided +# in the root validator — so these cases are built as full manifests, not bare `StoreEntry`s: the +# capability, its protocol, and a real process for `service` to resolve to (F5) all have to be in +# scope together for the rule to fire at all. +@pytest.mark.parametrize( + "process,protocol,sentinel,strategy", + [ + # redis's sentinel shape only pairs with datadir_copy or empty (§2b's catalog table). + (REDIS_PROCESS_EXAMPLE, "redis", {"key": "warm", "expected": "1"}, "template_database"), + # rabbitmq's sentinel shape only pairs with datadir_copy. + (RABBITMQ_PROCESS_EXAMPLE, "amqp", {"queue": "jobs", "expected_depth": 0}, "empty"), + ], + ids=["redis", "rabbitmq"], +) +def test_a_strategy_the_capabilitys_engine_does_not_support_is_rejected( + process: dict, protocol: str, sentinel: dict, strategy: str +) -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], process], + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "store": { + "protocol": protocol, + "service": process["name"], + "configuration_name": "STORE_URL", + }, + }, + "seed": { + "stores": [ + SEED_STORE_EXAMPLE, + { + "capability": "store", + "baseline": {"strategy": strategy, "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": sentinel, + }, + ] + }, + } + with pytest.raises(ValidationError, match="seed_strategy_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_postgres_excludes_the_empty_strategy() -> None: + """The one row of §2b's catalog table with no dedicated param above: postgres supports only + `template_database`/`datadir_copy`, unlike redis and rabbitmq which both accept a store with + no baseline state at all. Reuses the base manifest's own postgres/database pairing rather than + adding a process, since postgres already backs its only seed store.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": { + "stores": [ + { + **SEED_STORE_EXAMPLE, + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + } + ] + }, + } + with pytest.raises(ValidationError, match="seed_strategy_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_redis_capability_with_a_postgres_shaped_sentinel_is_rejected() -> None: + """The capability's protocol decides the engine, not the sentinel's own shape — a redis + capability paired with a postgres-shaped sentinel is a shape mismatch even though the sentinel + is internally well-formed and the strategy is one postgres would have accepted. The backing + process is genuinely redis (B1, p3-round2-review, requires protocol/engine agreement before + this check runs), so this stays isolated to the sentinel-shape question alone.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], REDIS_PROCESS_EXAMPLE], + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "protocol": "redis", + "service": "cache", + }, + }, + } + with pytest.raises(ValidationError, match="sentinel_shape_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_capability_protocol_that_disagrees_with_its_backing_process_engine_is_rejected() -> None: + """B1 (p3-round2-review): the sentinel and strategy below are both self-consistent with the + *declared* protocol (redis) — which is exactly what let this accept an invalid bundle before + the fix. The capability's `service` actually names the postgres process; only comparing the + protocol against `ManagedProcess.engine` catches it, since neither the sentinel shape nor the + strategy pairing is wrong on its own terms.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": {**FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "protocol": "redis"}, + }, + "seed": { + "stores": [ + { + "capability": "database", + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"key": "warm", "expected": "1"}, + } + ] + }, + } + with pytest.raises(ValidationError, match="capability_engine_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_on_an_unmapped_protocol_capability_is_rejected() -> None: + """B2 (p3-round2-review): before the fix, a store on a capability outside the + postgres/redis/amqp map made the engine lookup return `None` and the loop `continue`d, + skipping the sentinel and strategy checks entirely — a store on the `http` `tools` capability + validated with any sentinel and any strategy. It must now be rejected explicitly.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": { + "stores": [ + SEED_STORE_EXAMPLE, + { + "capability": "tools", + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"key": "anything", "expected": "1"}, + }, + ] + }, + } + with pytest.raises(ValidationError, match="store_protocol_unsupported"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_capability_engine_mismatch_is_caught_even_with_no_seed_store_at_all() -> None: + """F19 (p4-round1-review): before the fix, `capability_engine_mismatch` only ran inside the + `seed.stores` loop — a capability with no store entry at all (used only for a `{{...}}` + address, never seeded) was never compared, and could silently point `service` at the wrong + engine.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "cache": { + "protocol": "redis", "service": "postgres", "configuration_name": "CACHE_URL" + }, + }, + } + with pytest.raises(ValidationError, match="capability_engine_mismatch"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_store_whose_capability_service_is_a_source_process_is_rejected() -> None: + """F19 (p4-round1-review): a postgres-protocol store backed by a source process has no + managed engine to migrate or seed at all — previously accepted, since only a *wrong* managed + engine was checked, never a missing one.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], "service": "tools-api" + }, + }, + } + with pytest.raises(ValidationError, match="store_service_not_managed"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_postgres_capability_with_a_typod_redis_shaped_sentinel_is_rejected() -> None: + """A postgres capability whose sentinel was typo'd to redis's `{key, expected}` shape is + rejected naming the sentinel mismatch, not `seed_strategy_unsupported` against an engine + (`redis`) the bundle never declared — the bug this replaces would have named the wrong one.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "seed": {"stores": [{**SEED_STORE_EXAMPLE, "sentinel": {"key": "warm", "expected": "1"}}]}, + } + with pytest.raises(ValidationError, match="sentinel_shape_mismatch") as exc_info: + EnvironmentBundleV2.model_validate(manifest) + assert "seed_strategy_unsupported" not in str(exc_info.value) + + +def test_a_sentinel_mixing_two_protocol_shapes_is_rejected() -> None: + store = { + "capability": "database", + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1", "key": "also-set"}, + } + with pytest.raises(ValidationError, match="sentinel_shape_invalid"): + StoreEntry.model_validate(store) + + +def test_unknown_field_on_a_process_entry_is_rejected() -> None: + with pytest.raises(ValidationError, match="extra_forbidden"): + ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "mounts": ["/data"]}) + + +@pytest.mark.parametrize( + "bad_name", ["/etc", "../../etc", "Tools-Api", "tools_api!", "-leading-dash"], + ids=["absolute", "traversal", "uppercase", "punctuation", "leading-dash"], +) +def test_a_process_name_outside_the_closed_pattern_is_rejected(bad_name: str) -> None: + """F3, p5-round1-review — BLOCKER, model layer. `name` is path-joined into + `/work/build//` and `/work/worlds/w//` verbatim (§2b) — an unvalidated name + used to make `Path("/work/build") / "/etc"` collapse to `Path("/etc")`, which the provisioner + then `rmtree`'d as svc-control. Both process classes carry the same + `^[a-z0-9][a-z0-9_-]*$` pattern (§0 v1.8); `process_runtime.py`'s own `_ensure_within` is the + defense-in-depth backstop for a caller that bypasses this model layer entirely (see + `test_process_runtime.py`'s `test_build_tree_dir_rejects_a_name_that_escapes_the_work_directory` + and siblings — "tests both layers," per the worklist).""" + with pytest.raises(ValidationError, match="process_name_invalid"): + ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "name": bad_name}) + with pytest.raises(ValidationError, match="process_name_invalid"): + SourceProcess.model_validate({**AGENT_PROCESS_EXAMPLE, "name": bad_name}) + + +def test_process_names_matching_every_2b_example_are_accepted() -> None: + """Every example name in §2b's own JSON (`postgres`, `tools-api`, `agent`) must keep working — + the pattern is closed, not merely restrictive.""" + for name in ("postgres", "tools-api", "agent"): + assert ManagedProcess.model_validate({**POSTGRES_PROCESS_EXAMPLE, "name": name}).name == name + + +# --- §2a/§2b/§2d rules the model owns on its own, exercised at manifest scope ---------------- + + +def test_a_process_runtime_without_evidence_seam_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "runtime": {"kind": "process", "control_service": "agent"}} + with pytest.raises(ValidationError, match="evidence_seam_required"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_process_runtime_with_no_processes_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "processes": []} + with pytest.raises(ValidationError, match="processes_required"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_an_external_runtime_carrying_processes_or_seed_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "runtime": {"kind": "external"}} + with pytest.raises(ValidationError, match="processes_and_seed_forbidden"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_readiness_probe_naming_an_unresolved_capability_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "readiness": [{"capability": "does-not-exist"}]} + with pytest.raises(ValidationError, match="capability_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_duplicate_process_name_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [*FULL_MANIFEST_EXAMPLE["processes"], dict(POSTGRES_PROCESS_EXAMPLE)], + } + with pytest.raises(ValidationError, match="process_name_duplicate"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_capability_service_naming_an_unknown_process_is_rejected() -> None: + """The underscore/hyphen slip finding 5 names directly: a `service` of `tools_api` against a + process actually named `tools-api` must be caught here, not surface as an infrastructure + failure when the provisioner finds nothing to attach the capability to.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "tools": {**FULL_MANIFEST_EXAMPLE["capabilities"]["tools"], "service": "tools_api"}, + }, + } + with pytest.raises(ValidationError, match="service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_an_unresolved_control_service_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {**RUNTIME_EXAMPLE, "control_service": "not-a-process"}, + } + with pytest.raises(ValidationError, match="control_service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_control_service_resolving_to_a_managed_engine_is_rejected() -> None: + """N9 (p4-round2-review): `control_service` names the agent-side service the world handle and + evidence seam attach to (§2a) — a datastore in that role is incoherent. For THIS fixture, + pre-fix, the user-assignment loop below still caught it, just under the wrong code: `postgres` + got its expected `svc-data` (fine), but `agent` — carrying `user: "svc-agent"` from + `AGENT_PROCESS_EXAMPLE` — is no longer `control_service`, so the loop demands `svc-tools` for + it instead and raises `user_assignment_invalid: agent must be svc-tools, got svc-agent` (B1, + p4-round3-review). The genuine silent-load case needs a bundle where no source process claims + `svc-agent` at all (e.g. only `postgres` + `tools-api`) — every expectation is then satisfied + and the bundle loads with no agent-side process in it. This test still discriminates the fix + either way: the pre-fix code (`user_assignment_invalid`) does not match the post-fix code + (`control_service_unresolved`), so it goes red on a revert.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {**RUNTIME_EXAMPLE, "control_service": "postgres"}, + } + with pytest.raises(ValidationError, match="control_service_unresolved"): + EnvironmentBundleV2.model_validate(manifest) + + +# --- §2b/§0 user assignment (F5, p4-round1-review): control service -> svc-agent, other source -> +# svc-tools, managed engine -> svc-data. Decidable from the manifest's own fields once +# `control_service` is resolved. -------------------------------------------------------------- + + +def test_the_control_service_process_with_the_wrong_user_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + TOOLS_API_PROCESS_EXAMPLE, + {**AGENT_PROCESS_EXAMPLE, "user": "svc-tools"}, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_non_control_source_process_claiming_svc_agent_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + {**TOOLS_API_PROCESS_EXAMPLE, "user": "svc-agent"}, + AGENT_PROCESS_EXAMPLE, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_managed_engine_with_a_non_svc_data_user_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + {**POSTGRES_PROCESS_EXAMPLE, "user": "svc-tools"}, + TOOLS_API_PROCESS_EXAMPLE, + AGENT_PROCESS_EXAMPLE, + ], + } + with pytest.raises(ValidationError, match="user_assignment_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +# --- §2d configuration_name must not collide with the fixed placeholder vocabulary (F8, +# p4-round1-review) — a collision would render the builtin token instead of the capability's +# address, with no error and no way to spell the intended value. ------------------------------- + + +def test_a_configuration_name_matching_a_fixed_placeholder_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "configuration_name": "WORLD_DIR", + }, + }, + } + with pytest.raises(ValidationError, match="configuration_name_reserved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_configuration_name_matching_the_port_host_prefix_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "capabilities": { + **FULL_MANIFEST_EXAMPLE["capabilities"], + "database": { + **FULL_MANIFEST_EXAMPLE["capabilities"]["database"], + "configuration_name": "PORT_DB", + }, + }, + } + with pytest.raises(ValidationError, match="configuration_name_reserved"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_resolved_secret_value_in_process_environment_is_rejected() -> None: + """v1's manifest-level guard, reapplied: `environment`/`build_environment` are new in v2 and + are exactly where a resolved credential lands if an authoring stage inlines one instead of + routing it through `secret_purposes`.""" + manifest = { + **FULL_MANIFEST_EXAMPLE, + "processes": [ + POSTGRES_PROCESS_EXAMPLE, + TOOLS_API_PROCESS_EXAMPLE, + { + **AGENT_PROCESS_EXAMPLE, + "environment": { + **AGENT_PROCESS_EXAMPLE["environment"], + "STRIPE_API_KEY": "sk_live_x", + }, + }, + ], + } + with pytest.raises(ValidationError, match="resolved_secret_forbidden"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_an_external_runtime_with_a_capability_is_accepted() -> None: + """B3 (p3-round2-review): `kind: external` has no `processes` array to resolve capability + `service`/`control_service` against (§2a omits `processes` for it entirely) — the F5 + resolution checks must not run for it, or every external bundle carrying any capability at all + would be unloadable, which §2a never intended.""" + manifest = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "external-demo", + "runtime": {"kind": "external"}, + "capabilities": { + "target": { + "protocol": "http", + "service": "customer-endpoint", + "configuration_name": "TARGET_URL", + }, + }, + "provenance": {"source_kind": "remote", "source_digest": "c" * 64}, + } + bundle = EnvironmentBundleV2.model_validate(manifest) + assert bundle.runtime.kind.value == "external" + assert bundle.capabilities["target"].service == "customer-endpoint" + + +# --- F7 / B6: the digest-shape regexes, exercised on their rejection side too ---------------- + + +def test_a_non_hex_file_sha256_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "files": [{"path": "db/schema.sql", "sha256": "not-hex", "size": 10}], + } + with pytest.raises(ValidationError, match="file_sha256_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_non_hex_source_digest_is_rejected() -> None: + manifest = { + **FULL_MANIFEST_EXAMPLE, + "provenance": {**FULL_MANIFEST_EXAMPLE["provenance"], "source_digest": "not-hex"}, + } + with pytest.raises(ValidationError, match="source_digest_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_malformed_bundle_digest_is_rejected() -> None: + manifest = {**FULL_MANIFEST_EXAMPLE, "digest": "not-a-digest"} + with pytest.raises(ValidationError, match="bundle_digest_invalid"): + EnvironmentBundleV2.model_validate(manifest) + + +def test_a_malformed_inputs_digest_is_rejected() -> None: + store = { + **SEED_STORE_EXAMPLE, + "baseline": {"strategy": "template_database", "inputs_digest": "not-a-digest"}, + } + with pytest.raises(ValidationError, match="inputs_digest_invalid"): + StoreEntry.model_validate(store) + + +# --- §2c inputs_digest: byte-exact construction, checked against a hand-computed vector ----- + + +def test_compute_inputs_digest_matches_a_hand_computed_vector(tmp_path) -> None: + (tmp_path / "db").mkdir() + (tmp_path / "café").mkdir() + schema = b"CREATE TABLE riders (id int);" + # Multi-byte content pins content_length as a byte count, not a character count — reading via + # read_text() + len(text) would compute 8 here instead of 9 and still hash to something, just + # not this. Path with a non-ASCII segment pins the path encoding as utf-8, not latin-1. + seed = "-- café\n".encode("utf-8") + notes = b"-- see docs" + (tmp_path / "db" / "schema.sql").write_bytes(schema) + (tmp_path / "db" / "seed.sql").write_bytes(seed) + (tmp_path / "café" / "notes.sql").write_bytes(notes) + + # Built from §2c's own words, not by calling the helper: for each file in migrations then + # seed_files, in listed order, \n\n, then + # :\n. + expected = hashlib.sha256() + expected.update(b"db/schema.sql\n" + str(len(schema)).encode() + b"\n" + schema) + expected.update(b"db/seed.sql\n" + str(len(seed)).encode() + b"\n" + seed) + notes_path = "café/notes.sql".encode("utf-8") + expected.update(notes_path + b"\n" + str(len(notes)).encode() + b"\n" + notes) + expected.update(b"postgres:16\n") + + got = compute_inputs_digest( + tmp_path, + ["db/schema.sql"], + ["db/seed.sql", "café/notes.sql"], + engine=ManagedEngine.POSTGRES, + version="16", + ) + assert got == "sha256:" + expected.hexdigest() + + +def test_compute_inputs_digest_is_order_sensitive_not_sorted(tmp_path) -> None: + """Two migrations reversed must hash differently — order is part of the identity, since + migrations apply in sequence, not in whatever order sorting would put them in.""" + (tmp_path / "a.sql").write_bytes(b"A") + (tmp_path / "b.sql").write_bytes(b"B") + + forward = compute_inputs_digest( + tmp_path, ["a.sql", "b.sql"], [], engine=ManagedEngine.POSTGRES, version="16" + ) + backward = compute_inputs_digest( + tmp_path, ["b.sql", "a.sql"], [], engine=ManagedEngine.POSTGRES, version="16" + ) + assert forward != backward + + +# --- §2d bundle digest: byte-exact construction, checked against a hand-computed vector (F4, +# p4-round1-review) — the single normative implementation, `seal_bundle_v2`, over `BundleFileV2` +# directly, with no v1 model conversion. ------------------------------------------------------- + + +def test_seal_bundle_v2_matches_a_hand_computed_vector() -> None: + """Built from §2d's own words (v1.7), not by calling back into `seal_bundle_v2` or into + `json.dumps` with the same settings: sha256 over the canonical dump of the manifest minus + `digest`/`files`, then for each `files[]` record, in listed order, the canonical dump of + `{path, sha256, size}` prefixed by its byte length as 8 bytes big-endian. "Canonical" = + `json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False)`. The literal core + JSON below is transcribed from a minimal, otherwise-empty manifest's own normalized field set + (a hand-verified constant, not a derived one) — a real, if trivial, `external`-kind bundle + with no processes, no seed, no capabilities, and two files. A non-ASCII `name` pins + `ensure_ascii=False` (an `ensure_ascii=True` implementation would diverge here), and listing + `b.txt` before `a.txt` pins IN LISTED ORDER against an implementation that silently sorts + (N6, p4-round2-review).""" + file_sha_a = "b" * 64 + file_sha_b = "c" * 64 + manifest = EnvironmentBundleV2.model_validate( + { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "digest": "sha256:" + "0" * 64, + "name": "café", + "runtime": {"kind": "external"}, + "provenance": {"source_kind": "remote", "source_digest": "a" * 64}, + "files": [ + {"path": "b.txt", "sha256": file_sha_b, "size": 5}, + {"path": "a.txt", "sha256": file_sha_a, "size": 3}, + ], + } + ) + + core_json = ( + '{"capabilities":{},"metadata":{},"name":"café","processes":[],"provenance":' + '{"adopted_files":[],"commit":null,"generated_files":[],"generator":' + '"fi.alk.harness","generator_version":"1","repository":null,"source_digest":"' + + "a" * 64 + + '","source_kind":"remote"},"readiness":[],"runtime":' + '{"control_service":null,"document":null,"evidence_seam":null,"kind":"external"},' + '"schema_version":"futureagi.environment-bundle.v2","seed":null}' + ) + record_b_json = '{"path":"b.txt","sha256":"' + file_sha_b + '","size":5}' + record_a_json = '{"path":"a.txt","sha256":"' + file_sha_a + '","size":3}' + + expected = hashlib.sha256(core_json.encode("utf-8")) + for record_json in (record_b_json, record_a_json): + encoded = record_json.encode("utf-8") + expected.update(len(encoded).to_bytes(8, "big")) + expected.update(encoded) + + assert seal_bundle_v2(manifest) == "sha256:" + expected.hexdigest() + + +def test_seal_bundle_v2_ignores_the_manifests_own_digest_field() -> None: + """§2d: the digest is computed over the manifest minus `digest` and `files` — a manifest + whose only difference is its own (placeholder or stale) `digest` value must seal identically.""" + body = { + **FULL_MANIFEST_EXAMPLE, + "runtime": {"kind": "external"}, + "processes": [], + "seed": None, + "capabilities": {}, + "readiness": [], + } + a = EnvironmentBundleV2.model_validate({**body, "digest": "sha256:" + "0" * 64}) + b = EnvironmentBundleV2.model_validate({**body, "digest": "sha256:" + "1" * 64}) + assert seal_bundle_v2(a) == seal_bundle_v2(b) + + +# --- load_bundle_v2 -------------------------------------------------------------------------- + + +def test_load_bundle_v2_rejects_a_v1_manifest_with_a_typed_error(tmp_path) -> None: + (tmp_path / "manifest.json").write_text( + json.dumps({"schema_version": "futureagi.environment-bundle.v1"}) + ) + with pytest.raises(BundleV2Error, match="bundle_schema_unsupported"): + load_bundle_v2(tmp_path) + + +def test_load_bundle_v2_parses_a_valid_manifest_from_its_directory(tmp_path) -> None: + (tmp_path / "manifest.json").write_text(json.dumps(FULL_MANIFEST_EXAMPLE)) + bundle = load_bundle_v2(tmp_path) + assert bundle.name == "demo" diff --git a/tests/harness/test_call_runner.py b/tests/harness/test_call_runner.py new file mode 100644 index 00000000..dc1a8417 --- /dev/null +++ b/tests/harness/test_call_runner.py @@ -0,0 +1,799 @@ +"""`call_runner.py` against in-memory fakes and scripted `SimulationReport`s -- no real LiveKit +call, no real postgres, no monkeypatching LiveKit internals. Matches this repo's own convention +(`test_hosted_entrypoint.py`'s docstring): `asyncio.run` drives every `async def` seam directly. + +The test seam is `CallRunnerImpl`'s own boundary: the injectable `place_call(spec) -> +SimulationReport` callable named by the brief. Real production code (`_default_place_call`) +builds the same `SimulationSpec`/`SimulationRunner().run` pair; these tests never construct or +touch a real `SimulationRunner`, `rtc.Room`, or `AgentSession`. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from fi.alk.harness import call_runner as cr +from fi.alk.harness.bundle_v2 import EvidenceSeam +from fi.alk.harness.hosted_scheduler import CallAborted, CallOutcome +from fi.alk.harness.job import ( + AgentConnection, + ExecutionMode, + HarnessJob, + RepositorySource, + SourceKind, + SourceVisibility, +) +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeEndpoint, RuntimeState +from fi.alk.harness.world.errors import WorldUnavailable +from fi.alk.harness.world.runtime import Call +from fi.simulate.artifacts import ArtifactManifest +from fi.simulate.runtime.failures import FailureStage, SimulationFailure +from fi.simulate.runtime.report import SimulationReport, SimulationTestCaseResult +from fi.simulate.runtime.run import RunStatus +from fi.simulate.runtime.run import TestCaseStatus as CaseStatus +from fi.simulate.simulation.models import Persona as SimPersona +from fi.simulate.simulation.models import TestCaseResult as SimTestCaseResult + +LIVEKIT_API_KEY = "LIVEKIT_API_KEY" +LIVEKIT_API_SECRET = "LIVEKIT_API_SECRET" +DEEPGRAM_API_KEY = "DEEPGRAM_API_KEY" +GEMINI_API_KEY = "GEMINI_API_KEY" + +_ALL_SECRETS = { + LIVEKIT_API_KEY: "lk-key", + LIVEKIT_API_SECRET: "lk-secret", + DEEPGRAM_API_KEY: "dg-key", + GEMINI_API_KEY: "gm-key", +} +_ALL_CONFIG = {cr.LIVEKIT_URL_CONFIG_KEY: "wss://example.livekit.cloud"} + + +# ================================================================================================= +# Fixtures -- self-contained (this file touches nothing outside itself + call_runner.py). +# ================================================================================================= + + +def _job(*, connector: str = "livekit", config: dict[str, Any] | None = None) -> HarnessJob: + return HarnessJob( + job_id="job-abcdef12-xyz", + run_id="run-1", + execution=ExecutionMode.HOSTED, + source=RepositorySource( + kind=SourceKind.GITHUB, repository="org/repo", visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + ), + agent=AgentConnection(connector=connector, config=config or {}), + scenario_count=1, + seed=1, + ) + + +def _runtime( + *, + metadata: dict[str, Any] | None = None, + endpoints: dict[str, RuntimeEndpoint] | None = None, + world_index: int = 0, +) -> EnvironmentRuntime: + return EnvironmentRuntime( + runtime_id="rt-1", + world_index=world_index, + bundle_digest="sha256:" + "0" * 64, + state=RuntimeState.READY, + endpoints=endpoints or {}, + metadata=metadata or {}, + ) + + +def _postgres_endpoint(*, address: str = "postgresql://harness:pw@localhost:15001/w0") -> RuntimeEndpoint: + return RuntimeEndpoint( + capability="database", protocol="postgres", address=address, configuration_name="DATABASE_URL", + ) + + +def _context( + *, + tmp_path: Path, + connector: str = "livekit", + config: dict[str, Any] | None = None, + secrets: dict[str, str] | None = None, + evidence_seam: EvidenceSeam | None = EvidenceSeam.HTTP_TOOL, + attempt_number: int = 1, +) -> tuple[HarnessJob, cr.CallRunnerContext]: + job = _job(connector=connector, config=config if config is not None else dict(_ALL_CONFIG)) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir(parents=True, exist_ok=True) + context = cr.CallRunnerContext( + job=job, + bundle_dir=bundle_dir, + work_directory=tmp_path / "work", + evidence_seam=evidence_seam, + target_provider_secret_values=secrets if secrets is not None else dict(_ALL_SECRETS), + attempt_number=attempt_number, + ) + return job, context + + +def _write_scenario_doc( + bundle_dir: Path, + *, + scenario_key: str, + folder_name: str | None = None, + instruction: str = "Cancel order #42.", + persona: dict[str, Any] | None = None, + fixture: dict[str, Any] | None = None, + tests: str = "", +) -> None: + folder = bundle_dir / "scenarios" / (folder_name or scenario_key) + folder.mkdir(parents=True, exist_ok=True) + body = { + "scenario_key": scenario_key, + "scenario_id": "", + "sub_goals": [], + "instruction": instruction, + "persona": persona, + "fixture": fixture or {}, + "tests": tests, + } + (folder / "scenario.json").write_text(json.dumps(body), encoding="utf-8") + + +@dataclass +class _FakeScenario: + """`CallRunnerImpl.run` reads only `.scenario_key` off the scheduler's `Scenario` protocol.""" + + scenario_key: str + scenario_id: str = "" + + +@dataclass +class FakeAdapter: + """Narrow `ArtifactUploader` fake -- records every call, returns a real `sha256:` id + unless the kind is in `refuse_kinds` (mirrors the real adapter's null-not-crash contract).""" + + refuse_kinds: frozenset = field(default_factory=frozenset) + uploads: list[tuple[Any, str | None, bytes]] = field(default_factory=list) + + async def upload_artifact(self, data: bytes, *, kind, scenario_key=None, deadline=None) -> str | None: + if kind in self.refuse_kinds: + return None + import hashlib + + digest = hashlib.sha256(data).hexdigest() + self.uploads.append((kind, scenario_key, data)) + return f"sha256:{digest}" + + +def _persona() -> SimPersona: + return SimPersona(persona={"name": "customer"}, situation="s", outcome="o") + + +def _report( + *, + status: RunStatus = RunStatus.COMPLETED, + case_status: CaseStatus | None = CaseStatus.COMPLETED, + transcript: str = "hello there", + messages: list[dict[str, str]] | None = None, + failure: SimulationFailure | None = None, + started_at: datetime | None = None, + ended_at: datetime | None = None, + no_cases: bool = False, + run_id: str = "sim-run-1", +) -> SimulationReport: + messages = messages if messages is not None else [{"role": "user", "content": "hi"}] + started_at = started_at or datetime.now(timezone.utc) + ended_at = ended_at or (started_at + timedelta(seconds=30)) + cases: list[SimulationTestCaseResult] = [] + if not no_cases: + assert case_status is not None + result = SimTestCaseResult(persona=_persona(), transcript=transcript, messages=messages) + cases.append( + SimulationTestCaseResult( + test_case_id="tc-1", status=case_status, persona=_persona(), result=result, + failure=failure, started_at=started_at, ended_at=ended_at, + ) + ) + return SimulationReport( + run_id=run_id, spec_hash="hash", status=status, started_at=started_at, ended_at=ended_at, + test_cases=cases, artifacts=ArtifactManifest(run_id=run_id), + ) + + +def _run(runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + return asyncio.run(runner.run(scenario, runtime)) + + +def _run_expect_abort( + runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime +) -> CallAborted: + try: + asyncio.run(runner.run(scenario, runtime)) + except CallAborted as exc: + return exc + raise AssertionError("expected CallAborted, nothing was raised") + + +def _run_expect_world_unavailable( + runner: cr.CallRunnerImpl, scenario: _FakeScenario, runtime: EnvironmentRuntime +) -> WorldUnavailable: + try: + asyncio.run(runner.run(scenario, runtime)) + except WorldUnavailable as exc: + return exc + raise AssertionError("expected WorldUnavailable, nothing was raised") + + +# ================================================================================================= +# Room naming (deterministic scheme, pinned verbatim by this file). WHY only a prefix at the wire: +# engines/livekit.py::_resolve_room_name appends its own `-{invocation_id}-{test_case_id[-12:]}` +# suffix in managed room_mode unless `room_name_verbatim` is set (this runner does not set it), so +# the pinned string below is the deterministic PREFIX every dialed room carries, not the full +# on-the-wire room name. +# ================================================================================================= + + +def test_room_name_matches_the_pinned_deterministic_scheme() -> None: + name = cr._room_name(job_id="abcdef1234567890", attempt_number=2, scenario_key="cancel-order", scenario_attempt=3) + assert name == "harness-abcdef12-a2-cancel-order-s3" + + +def test_room_name_uses_only_the_first_eight_chars_of_job_id() -> None: + short = cr._room_name(job_id="ab", attempt_number=1, scenario_key="k", scenario_attempt=1) + assert short == "harness-ab-a1-k-s1" + + +# ================================================================================================= +# Pre-dial validation. +# ================================================================================================= + + +def test_missing_target_provider_secrets_aborts_pre_dial_without_calling_place_call(tmp_path: Path) -> None: + called = False + + async def place_call(spec): + nonlocal called + called = True + raise AssertionError("place_call must never be reached") + + _job_obj, context = _context(tmp_path=tmp_path, secrets={}) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is None + assert str(exc).startswith("voice_capability_unavailable: missing") + assert LIVEKIT_API_KEY in str(exc) + assert not called + + +def test_missing_llm_credential_names_the_either_or_pair(tmp_path: Path) -> None: + secrets = dict(_ALL_SECRETS) + del secrets[GEMINI_API_KEY] + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "GEMINI_API_KEY_or_GOOGLE_API_KEY" in str(exc) + + +def test_google_api_key_alone_satisfies_the_llm_credential_check(tmp_path: Path) -> None: + secrets = dict(_ALL_SECRETS) + del secrets[GEMINI_API_KEY] + secrets["GOOGLE_API_KEY"] = "g-key" + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + assert runner._missing_config is None + + +def test_missing_livekit_url_config_aborts_pre_dial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path, config={}) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "config=livekit_url" in str(exc) + + +def test_missing_dispatch_identity_metadata_aborts_pre_dial(tmp_path: Path) -> None: + """Verified finding: `EnvironmentRuntime.metadata` is always `{}` in this worktree's HEAD -- + this is the realistic default a real hosted run hits today (CONTRACT NOTE 3).""" + called = False + + async def place_call(spec): + nonlocal called + called = True + raise AssertionError("place_call must never be reached") + + _job_obj, context = _context(tmp_path=tmp_path) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={})) + assert "voice_dispatch_identity_unavailable" in str(exc) + assert "livekit_agent_name" in str(exc) + assert not called + + +def test_missing_scenario_document_aborts_pre_dial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + runner = cr.CallRunnerImpl(FakeAdapter(), context) + exc = _run_expect_abort( + runner, _FakeScenario("no-such-key"), _runtime(metadata={"livekit_agent_name": "agent-w0"}) + ) + assert "voice_scenario_document_unavailable" in str(exc) + + +def test_scenario_document_matched_by_scenario_key_field_not_folder_name(tmp_path: Path) -> None: + """scenario_source.py's own convention: `scenario_key` is a field INSIDE scenario.json, not + necessarily the folder name -- this runner must match the same way, not assume they agree.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="the-real-key", folder_name="some-other-folder-name") + + captured: dict[str, Any] = {} + + async def place_call(spec): + captured["spec"] = spec + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + _run(runner, _FakeScenario("the-real-key"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert captured["spec"] is not None + + +# ================================================================================================= +# Happy path: COMPLETED -> real CallOutcome, artifacts uploaded, dispatch/room wiring correct. +# ================================================================================================= + + +def test_completed_call_uploads_transcript_and_returns_populated_outcome(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=EvidenceSeam.HTTP_TOOL) + _write_scenario_doc(context.bundle_dir, scenario_key="k1", instruction="Cancel order #42.") + + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended = started + timedelta(seconds=45) + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}] + + async def place_call(spec): + return _report(transcript="hi\nhello", messages=messages, started_at=started, ended_at=ended) + + adapter = FakeAdapter() + runner = cr.CallRunnerImpl(adapter, context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 2 + assert outcome.duration_ms == 45_000 + assert outcome.transcript_artifact is not None + assert outcome.transcript_artifact.startswith("sha256:") + assert outcome.calls == () # http_tool: STOPPED, always zero -- CONTRACT NOTE 1 + assert len(adapter.uploads) == 1 + + +def test_dispatch_agent_name_and_livekit_url_flow_into_the_built_spec(tmp_path: Path) -> None: + _job_obj, context = _context( + tmp_path=tmp_path, config={cr.LIVEKIT_URL_CONFIG_KEY: "wss://custom.livekit.cloud"}, + ) + _write_scenario_doc(context.bundle_dir, scenario_key="k1", instruction="Do the thing.") + + captured: dict[str, Any] = {} + + async def place_call(spec): + captured["spec"] = spec + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + + spec = captured["spec"] + agent_definition = spec.environment.config["agent_definition"] + livekit_runtime = spec.environment.config["livekit_runtime"] + assert agent_definition["agent_name"] == "agent-w0" + assert agent_definition["system_prompt"] == "Do the thing." + # WHY prefix-match, not exact-match: this asserts what THIS runner puts on the spec, which is + # exactly the pinned deterministic scheme -- but engines/livekit.py::_resolve_room_name appends + # its own suffix in managed room_mode before the room is actually dialed, so a reader must not + # take this string as the full on-the-wire room name. + assert livekit_runtime["room_name"].startswith("harness-job-abcd-a1-k1-s1") + assert livekit_runtime["url"] == "wss://custom.livekit.cloud/" + + +def test_scenario_attempt_counter_increments_per_scenario_key_across_retries(tmp_path: Path) -> None: + """The scheduler retries the SAME scenario_key (e.g. after evidence_missing) -- successive + `run()` calls for one key must get distinct room names, or a LiveKit room collision follows.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + _write_scenario_doc(context.bundle_dir, scenario_key="k2") + + rooms: list[str] = [] + + async def place_call(spec): + rooms.append(spec.environment.config["livekit_runtime"]["room_name"]) + return _report() + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + runtime = _runtime(metadata={"livekit_agent_name": "agent-w0"}) + _run(runner, _FakeScenario("k1"), runtime) + _run(runner, _FakeScenario("k2"), runtime) + _run(runner, _FakeScenario("k1"), runtime) + + assert rooms[0].endswith("-k1-s1") + assert rooms[1].endswith("-k2-s1") + assert rooms[2].endswith("-k1-s2") + + +# ================================================================================================= +# Failure semantics -- the three cases the brief pins. +# ================================================================================================= + + +def test_agent_unavailable_status_raises_world_unavailable(tmp_path: Path) -> None: + """Verified against engines/livekit.py: AGENT_UNAVAILABLE fires ONLY on a readiness-stage + timeout with a session started but no target dispatched -- exactly "dispatch fails, agent + never joins," matching the brief's explicit WorldUnavailable case.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.READINESS, code="agent_unavailable", + message="Target agent did not become ready", + ) + return _report(case_status=CaseStatus.AGENT_UNAVAILABLE, failure=failure) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_world_unavailable( + runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"}) + ) + assert "Target agent did not become ready" in str(exc) + + +def test_non_completed_status_raises_call_aborted_with_partial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + started = datetime(2026, 1, 1, tzinfo=timezone.utc) + ended = started + timedelta(seconds=12) + + async def place_call(spec): + failure = SimulationFailure(stage=FailureStage.RUNNING, code="case_execution_error", message="boom") + return _report( + case_status=CaseStatus.FAILED, failure=failure, started_at=started, ended_at=ended, + transcript="", messages=[], + ) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.duration_ms == 12_000 + assert exc.partial.calls == () + assert "boom" in str(exc) + + +def test_no_test_cases_in_report_raises_call_aborted_with_timing_only_partial(tmp_path: Path) -> None: + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + return _report(status=RunStatus.TIMED_OUT, no_cases=True) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.turns == 0 + assert exc.partial.calls == () + + +def test_place_call_exception_raises_call_aborted_with_timing_partial_never_raw(tmp_path: Path) -> None: + """world-handle-interface.md's partial-call rule: a generic exception must never lose timing + (the brief: "never let a raw exception escape post-dial").""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + raise RuntimeError("engine exploded") + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.started_at is not None + assert exc.partial.ended_at is not None + assert "engine exploded" in str(exc) + + +def test_translate_report_upload_failure_raises_call_aborted_with_timing_partial_never_raw( + tmp_path: Path, +) -> None: + """The same partial-call rule as `place_call` failing above, but for a failure INSIDE + `_translate_report` itself (a transcript/recording read or an `upload_artifact` surprise) -- + this must also never escape `run()` raw and lose the timing the call already measured.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + class RaisingAdapter: + async def upload_artifact(self, data, *, kind, scenario_key=None, deadline=None): + raise RuntimeError("upload exploded") + + async def place_call(spec): + return _report( + transcript="hi", + messages=[{"role": "user", "content": "hi"}, {"role": "assistant", "content": "yo"}], + ) + + runner = cr.CallRunnerImpl(RaisingAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.started_at is not None + assert exc.partial.ended_at is not None + assert exc.partial.calls == () + assert "upload exploded" in str(exc) + + +def test_place_call_outer_timeout_raises_call_aborted_with_timing_partial( + tmp_path: Path, monkeypatch +) -> None: + """Forces the runner-owned `asyncio.wait_for` to actually fire (not just the SDK's own + internal one) by shrinking every phase-overhead constant to a few milliseconds -- avoids a + multi-minute real sleep in the test suite while still exercising the real timeout code path.""" + monkeypatch.setattr(cr, "_CONNECT_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_READINESS_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_CLEANUP_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(cr, "_RUN_SECONDS_PAD_SECONDS", 0.01) + monkeypatch.setattr(cr, "_OUTER_WAIT_FOR_PAD_SECONDS", 0.01) + + _job_obj, context = _context( + tmp_path=tmp_path, config={cr.LIVEKIT_URL_CONFIG_KEY: "wss://x", cr.CALL_TIMEOUT_CONFIG_KEY: 0.01}, + ) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + await asyncio.Event().wait() # never completes on its own + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert "voice_call_runner_timeout" in str(exc) + assert exc.partial is not None + assert exc.partial.calls == () + + +def test_silent_agent_real_engine_shape_returns_normal_outcome_with_empty_calls(tmp_path: Path) -> None: + """Pins the shape the real engine actually produces for a silent agent-first call + (engines/livekit.py::_conversation_outcome): status FAILED, code "no_conversation", zero + messages -- never a COMPLETED case with zero turns (COMPLETED requires >= min_turn_messages + and role alternation, a shape the engine cannot produce for a silent call). Must still surface + as a NORMAL CallOutcome with the real (zero) turn count, never a WorldUnavailable/CallAborted, + letting the scheduler's own coverage guarantee turn it into evidence_missing.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="no_conversation", + message="No conversation turns were committed before the inactivity deadline", + retryable=True, + ) + return _report(case_status=CaseStatus.FAILED, failure=failure, transcript="", messages=[]) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 0 + assert outcome.calls == () + assert outcome.transcript_artifact is None # empty transcript -- never uploaded + + +def test_silent_agent_conversation_silence_timeout_code_also_returns_normal_outcome(tmp_path: Path) -> None: + """The engine's other zero-turn silent code (the agent-first silence watchdog firing before + any turn ever lands) must map the same way as "no_conversation" above.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="conversation_silence_timeout", + message="Agent-first conversation stalled after it began", retryable=True, + ) + return _report(case_status=CaseStatus.FAILED, failure=failure, transcript="", messages=[]) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert isinstance(outcome, CallOutcome) + assert outcome.turns == 0 + assert outcome.calls == () + + +def test_silent_agent_mapping_is_scoped_to_zero_turns_only(tmp_path: Path) -> None: + """A short-but-nonzero conversation carrying the same failure code must NOT be laundered into + a normal outcome -- only a genuinely zero-turn silent call qualifies; this stays a CallAborted + exactly like any other non-completed status.""" + _job_obj, context = _context(tmp_path=tmp_path) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + messages = [{"role": "user", "content": "hi"}, {"role": "assistant", "content": "hello"}] + + async def place_call(spec): + failure = SimulationFailure( + stage=FailureStage.RUNNING, code="conversation_silence_timeout", + message="Agent-first conversation stalled after it began", retryable=True, + ) + return _report( + case_status=CaseStatus.FAILED, failure=failure, transcript="hi\nhello", messages=messages, + ) + + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + exc = _run_expect_abort(runner, _FakeScenario("k1"), _runtime(metadata={"livekit_agent_name": "agent-w0"})) + assert exc.partial is not None + assert exc.partial.turns == 2 + + +# ================================================================================================= +# Evidence collection. +# ================================================================================================= + + +def test_http_tool_seam_always_returns_no_calls() -> None: + """CONTRACT NOTE 1: STOPPED, verified -- no invented capture proxy.""" + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + assert cr._collect_http_tool_calls(runtime) == () + + +def test_find_postgres_endpoint_matches_by_protocol_not_a_fixed_slug_name() -> None: + """Corrects the brief's literal `runtime.endpoints["database"]` wording: capability slugs are + bundle-author-chosen (verified against `build_endpoints`/`_find_postgres_endpoint` in + hosted_entrypoint.py) -- a bundle naming its slug anything else must still resolve.""" + endpoint = _postgres_endpoint() + runtime = _runtime(endpoints={"orders_store": endpoint}) + found = cr._find_postgres_endpoint(runtime) + assert found is endpoint + + +def test_find_postgres_endpoint_returns_none_when_absent() -> None: + runtime = _runtime(endpoints={"queue": RuntimeEndpoint(capability="queue", protocol="amqp", address="amqp://x")}) + assert cr._find_postgres_endpoint(runtime) is None + + +class _FakeCursor: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self.description = [(name,) for name in columns] + + def fetchall(self) -> list[tuple[Any, ...]]: + return self._rows + + +class _FakeConnection: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self._columns = columns + self.executed: list[str] = [] + + def execute(self, statement: str, params: Any = None) -> _FakeCursor: + self.executed.append(statement) + return _FakeCursor(self._rows, self._columns) + + def __enter__(self) -> "_FakeConnection": + return self + + def __exit__(self, *exc_info: Any) -> bool: + return False + + +class _FakePsycopg: + def __init__(self, rows: list[tuple[Any, ...]], columns: list[str]) -> None: + self._rows = rows + self._columns = columns + self.connections: list[_FakeConnection] = [] + + def connect(self, dsn: str, **kwargs: Any) -> _FakeConnection: + connection = _FakeConnection(self._rows, self._columns) + self.connections.append(connection) + return connection + + +class _RaisingPsycopg: + class Error(Exception): + pass + + def connect(self, dsn: str, **kwargs: Any) -> Any: + raise self.Error("connection refused") + + +def test_tool_trace_translates_rows_and_applies_v1_refused_rule(monkeypatch) -> None: + """world-handle-interface.md V1 rule: `refused = not ok` (a trace cannot distinguish refusal + from crash).""" + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [ + ("lookup_order", {"id": "1"}, {"status": "shipped"}, True, "", 100.5), + ("cancel_order", {"id": "2"}, None, False, "not found", 101.0), + ] + fake = _FakePsycopg(rows, columns) + monkeypatch.setitem(sys.modules, "psycopg", fake) + + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + + assert calls == ( + Call(name="lookup_order", arguments={"id": "1"}, result={"status": "shipped"}, ok=True, error="", refused=False, at=100.5), + Call(name="cancel_order", arguments={"id": "2"}, result=None, ok=False, error="not found", refused=True, at=101.0), + ) + + +def test_tool_trace_read_failure_degrades_to_no_calls_never_crashes(monkeypatch) -> None: + """No producer exists yet (CONTRACT NOTE 2) -- a missing table / connection failure must + degrade to `()`, never raise past this function.""" + monkeypatch.setitem(sys.modules, "psycopg", _RaisingPsycopg()) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + assert cr._collect_tool_trace_calls(runtime) == () + + +def test_tool_trace_missing_endpoint_degrades_to_no_calls() -> None: + runtime = _runtime(endpoints={}) + assert cr._collect_tool_trace_calls(runtime) == () + + +def test_tool_trace_result_string_form_is_truncated_at_2000_chars(monkeypatch) -> None: + long_string = "x" * 3000 + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("t", {}, long_string, True, "", 1.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + assert len(calls) == 1 + assert len(calls[0].result) == 2000 + + +def test_tool_trace_parsed_json_result_is_not_truncated(monkeypatch) -> None: + """Per world-handle-interface.md: "result — parsed JSON where the source captured JSON, else + a string; both result (string form) and error truncated at 2,000 chars" -- truncation applies + to the STRING form only.""" + big_list = list(range(3000)) + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("t", {}, big_list, True, "", 1.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + runtime = _runtime(endpoints={"database": _postgres_endpoint()}) + calls = cr._collect_tool_trace_calls(runtime) + assert calls[0].result == big_list + + +def test_tool_trace_clear_is_best_effort_and_never_raises(monkeypatch) -> None: + monkeypatch.setitem(sys.modules, "psycopg", _RaisingPsycopg()) + cr._clear_tool_trace_calls("postgresql://x/y") # must not raise + + +def test_completed_call_with_tool_trace_seam_collects_evidence(tmp_path: Path, monkeypatch) -> None: + columns = ["name", "arguments", "result", "ok", "error", "at"] + rows = [("do_thing", {}, "done", True, "", 5.0)] + monkeypatch.setitem(sys.modules, "psycopg", _FakePsycopg(rows, columns)) + + _job_obj, context = _context(tmp_path=tmp_path, evidence_seam=EvidenceSeam.TOOL_TRACE) + _write_scenario_doc(context.bundle_dir, scenario_key="k1") + + async def place_call(spec): + return _report() + + runtime = _runtime( + metadata={"livekit_agent_name": "agent-w0"}, endpoints={"database": _postgres_endpoint()}, + ) + runner = cr.CallRunnerImpl(FakeAdapter(), context, place_call=place_call) + outcome = _run(runner, _FakeScenario("k1"), runtime) + assert len(outcome.calls) == 1 + assert outcome.calls[0].name == "do_thing" + + +# ================================================================================================= +# Credential export (WHY: the LiveKit engine reads these via ambient os.environ, not spec fields). +# ================================================================================================= + + +def test_construction_exports_target_provider_secrets_to_environ_once(tmp_path: Path) -> None: + fake_environ: dict[str, str] = {} + _job_obj, context = _context(tmp_path=tmp_path) + cr.CallRunnerImpl(FakeAdapter(), context, environ=fake_environ) + assert fake_environ[LIVEKIT_API_KEY] == "lk-key" + assert fake_environ[LIVEKIT_API_SECRET] == "lk-secret" + assert fake_environ[DEEPGRAM_API_KEY] == "dg-key" + assert fake_environ[GEMINI_API_KEY] == "gm-key" + + +def test_construction_never_exports_secrets_outside_the_target_provider_map(tmp_path: Path) -> None: + fake_environ: dict[str, str] = {} + secrets = dict(_ALL_SECRETS) + secrets["UNRELATED_ALIAS"] = "should-not-export" + _job_obj, context = _context(tmp_path=tmp_path, secrets=secrets) + cr.CallRunnerImpl(FakeAdapter(), context, environ=fake_environ) + assert "UNRELATED_ALIAS" not in fake_environ diff --git a/tests/harness/test_hosted_entrypoint.py b/tests/harness/test_hosted_entrypoint.py new file mode 100644 index 00000000..e330d624 --- /dev/null +++ b/tests/harness/test_hosted_entrypoint.py @@ -0,0 +1,3066 @@ +"""`hosted_entrypoint.py` against in-memory fakes — no real postgres, no real network. + +`asyncio.run` drives every `async def` seam here, matching `test_hosted_scheduler.py`'s own +convention (no pytest-asyncio dependency in this repo). Verification for this file was done by +importing it and calling each `test_*` function directly, not via a `pytest` +invocation. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import hashlib +import json +import os +import random +import stat +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable +from unittest import mock + +from fi.alk.harness import hosted_entrypoint as he +from fi.alk.harness import outbound as ob +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + EnvironmentBundleV2, + EvidenceSeam, + ManagedEngine, + compute_inputs_digest, + seal_bundle_v2, +) +from fi.alk.harness.hosted_scheduler import ( + Call, + CallAborted, + CallOutcome, + HostedScheduler, + ReceiptFailure, + ResultReceipt, + RunResult, + WorldPool, +) +from fi.alk.harness.job import ( + AgentConnection, + ArtifactLevel, + ExecutionMode, + FailureDomain, + HarnessArtifactPolicy, + HarnessJob, + HarnessStage, + RepositorySource, + SourceKind, + SourceVisibility, +) +from fi.alk.harness.process_runtime import ( + EnvironmentRuntime, + ProcessRuntimeError, + RuntimeEndpoint, + RuntimeState, +) +from fi.alk.harness.scenario_source import SCENARIOS_DIRNAME +from fi.simulate.runtime.spec import RuntimeRequirements, SecretRef + +SCHEMA_SQL = b"CREATE TABLE riders (id int);\n" +SEED_SQL = b"INSERT INTO riders VALUES (1);\n" +TARGET_PROVIDER_ALIAS = "LIVEKIT_API_KEY" + + +# ================================================================================================= +# Bundle fixture — mirrors test_process_preflight.py's own helper (not imported: this file is +# self-contained per the "touch only your two new files" rule). +# ================================================================================================= + + +def _base_manifest_body() -> dict[str, Any]: + return { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": [], + }, + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], "user": "svc-agent", "depends_on": ["postgres"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", "service": "postgres", "configuration_name": "DATABASE_URL", + }, + }, + "readiness": [], + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64, + }, + "metadata": {}, + } + + +def _write_bundle(root: Path) -> EnvironmentBundleV2: + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL} + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", "migrations": ["db/schema.sql"], "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +def _write_storeless_bundle(root: Path) -> EnvironmentBundleV2: + """A legal stateless-agent bundle: no managed postgres process, zero capabilities, no seed. + What the relaxed store rule admits — the run below must work end-to-end against it.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + body["processes"] = [ + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": {"LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}"}, + "secret_purposes": ["target_provider"], "user": "svc-agent", "depends_on": [], + }, + ] + body["capabilities"] = {} + body["files"] = [] + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +def _job( + *, connector: str = "vapi", parallelism: int = 1, + artifacts: HarnessArtifactPolicy | None = None, +) -> HarnessJob: + return HarnessJob( + job_id="job-1", run_id="run-1", execution=ExecutionMode.HOSTED, + source=RepositorySource( + kind=SourceKind.GITHUB, repository="org/repo", visibility=SourceVisibility.PUBLIC, + commit_sha="a" * 40, + ), + agent=AgentConnection( + connector=connector, + secret_refs={ + TARGET_PROVIDER_ALIAS: SecretRef( + manager="platform-vault", key="secret-id", purpose="target_provider" + ) + }, + ), + scenario_count=2, + seed=1234, + runtime=RuntimeRequirements(parallelism=parallelism), + **({"artifacts": artifacts} if artifacts is not None else {}), + ) + + +def _write_job(path: Path, job: HarnessJob) -> None: + path.write_text(job.model_dump_json(), encoding="utf-8") + + +# ================================================================================================= +# Capabilities fixture. +# ================================================================================================= + + +def _capabilities(*, attempt_id: str = "attempt-1") -> ob.HostedCapabilities: + base = f"https://platform.example/simulate/api/harness/attempts/{attempt_id}" + return ob.HostedCapabilities.model_validate( + { + "schema_version": ob.CAPABILITIES_SCHEMA_VERSION, + "job_id": "job-1", + "attempt_id": attempt_id, + "attempt_number": 1, + "fence": "fence-1", + "expires_at": "2999-01-01T00:00:00.000Z", + "token": "bearer-token", + "endpoints": { + "events": f"{base}/events/", + "results": f"{base}/results/", + "artifacts": f"{base}/artifacts/", + "scenarios": f"{base}/scenarios/", + }, + } + ) + + +# ================================================================================================= +# FakeTransport — a minimal in-memory platform. Routes on a URL substring, not a full router: this +# module only needs the four channel shapes, not a general HTTP mock. +# ================================================================================================= + + +@dataclass +class FakeTransport: + fence_after: int | None = None # 1-based call count at which every further call 403s. + fence_on_url_substring: str | None = None # once a URL matches, that call and every later one 403s. + # fences the specific events POST whose batch carries a `type: "terminal"` record -- + # `emit_terminal()`'s own spool append succeeds, so this reproduces the fence landing on the + # network flush that delivers the terminal event, not before it. + fence_on_terminal_event: bool = False + _fenced: bool = False + calls: list[dict[str, Any]] = field(default_factory=list) + event_records: list[dict[str, Any]] = field(default_factory=list) + receipts: dict[tuple[str, str], dict[str, Any]] = field(default_factory=dict) + artifacts: dict[str, bytes] = field(default_factory=dict) + manifests: list[dict[str, Any]] = field(default_factory=list) + scenarios_calls: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + del timeout + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + if self.fence_on_url_substring is not None and self.fence_on_url_substring in url: + self._fenced = True + if ( + self.fence_on_terminal_event + and "/events/" in url + and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + self._fenced = True + if self._fenced or (self.fence_after is not None and len(self.calls) >= self.fence_after): + return ob.TransportResponse( + status_code=403, + body={"error": "fenced", "message": "attempt superseded", "retryable": False}, + headers={}, + ) + if "/events/" in url and method == "POST": + body_bytes = data if isinstance(data, (bytes, bytearray)) else b"" + body = json.loads(body_bytes.decode("utf-8")) if body_bytes else {"events": []} + events = body.get("events", []) + self.event_records.extend(events) + watermark = max((e["sequence"] for e in events), default=0) + return ob.TransportResponse(200, {"acked_through_sequence": watermark, "rejected": []}, {}) + if "/results/" in url and method == "POST" and json_body is not None: + key = (json_body["job_id"], json_body["scenario_key"]) + existed = key in self.receipts + self.receipts[key] = json_body + return ob.TransportResponse(200 if existed else 201, {}, {}) + if url.endswith("/manifest/") and method == "POST" and json_body is not None: + self.manifests.append(json_body) + return ob.TransportResponse(200, {}, {}) + if "/artifacts/" in url and method == "PUT": + digest = url.rstrip("/").rsplit("/", 1)[-1] + payload = data if isinstance(data, (bytes, bytearray)) else b"".join(data) + existed = digest in self.artifacts + self.artifacts[digest] = bytes(payload) + return ob.TransportResponse(200 if existed else 201, {}, {}) + if "/scenarios/" in url and method == "POST" and json_body is not None: + # p13: Azain's real router mints exactly ONE url per attempt (a DRF detail `@action`, + # no `url_path`) -- provision vs begin is a body-level `operation` field, never a URL + # suffix, so routing here is on `json_body["operation"]`, not `url`. + self.scenarios_calls.append((url, json_body)) + operation = json_body.get("operation") + if operation == "provision": + keys = [p.get("scenario_key") for p in json_body.get("personas", [])] + scenarios = [{"scenario_key": key, "scenario_id": f"platform-{key}"} for key in keys] + return ob.TransportResponse( + 200, {"result": {"run_test_id": "run-test-1", "scenarios": scenarios}}, {} + ) + if operation == "begin": + return ob.TransportResponse( + 200, {"result": {"test_execution_id": "exec-1", "scenarios": []}}, {} + ) + # No/unknown `operation` -- exercised by callers (e.g. `FakeScenarioSource`) that only + # care about the call happening, never the response shape. + return ob.TransportResponse(200, {"result": {"ok": True}}, {}) + return ob.TransportResponse( + 404, {"error": "not_found", "message": f"unmapped route: {url}", "retryable": False}, {} + ) + + def terminal_events(self) -> list[dict[str, Any]]: + return [record for record in self.event_records if record.get("type") == "terminal"] + + +# ================================================================================================= +# Fake provisioner / world / scenario / call runner. +# ================================================================================================= + + +class FakeProvisioner: + """`_serialized` records any overlapping call into `overlaps` rather than asserting in-band -- + an in-band `assert not self._busy` fires INSIDE `WorldPool.lease()`'s own + `except Exception as exc: reset_exc = exc` (a reset/healthy failure is expected there) or + `_reconcile`'s `except Exception` (a reconcile must never crash the pool), so production + swallows it and a caller driving this fake through `WorldPool` (rather than calling it + directly) would never see the failure. Asserting on `overlaps` from the test's OWN frame is + what actually makes a `_provider_lock` regression observable end-to-end (R7: this is the same + fake `WorldPool`'s own test suite already exercises this way, now needed here too since + `hosted_entrypoint.py` no longer wraps the provider in its own lock).""" + + name = "fake-process" # mirrors `ProcessRuntimeProvider.name`. + + def __init__(self, instances: int = 1, *, always_unhealthy: bool = False) -> None: + self.instances = instances + self.always_unhealthy = always_unhealthy # every healthy() probe fails. + self.provision_calls = 0 + self.reset_calls = 0 + self.healthy_calls = 0 + self.closed = False + self.overlaps: list[str] = [] + self._in_flight: list[str] = [] + self._runtimes = { + i: EnvironmentRuntime( + runtime_id=f"digest:w{i}", world_index=i, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + for i in range(instances) + } + + @contextlib.asynccontextmanager + async def _serialized(self, label: str): + if self._in_flight: + self.overlaps.append(f"{self._in_flight[-1]} overlapped {label}") + self._in_flight.append(label) + try: + await asyncio.sleep(0) + yield + finally: + self._in_flight.remove(label) + + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract + async with self._serialized("provision"): + self.provision_calls += 1 + return [self._runtimes[i] for i in range(instances)] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + del work_directory + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + runtime.state = RuntimeState.READY + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + del work_directory + # v1.12 folds `healthy` into the same non-reentrant set as provision/reset/close. + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return not self.always_unhealthy + + async def close(self, *, work_directory: Path) -> None: + del work_directory + async with self._serialized("close"): + self.closed = True + + +class FakeWorld: + def __init__(self, world_index: int, rng: Any) -> None: + self.world_index = world_index + self.rng = rng + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + del table + return {} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + del collection, key + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + del collection, key, changes, by + return 1 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + del collection, key, by + return 1 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + del sql, params + return [] + + def read_only(self) -> "FakeWorld": + return self + + +class FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: Any) -> FakeWorld: + return FakeWorld(runtime.world_index, rng) + + +@dataclass +class FakeSubGoal: + name: str + should_hold: bool + judged: str = "yes" + + def check(self, world: Any, calls: Any) -> object: + del world, calls + return None if self.should_hold else "the agent did not do it" + + +@dataclass +class FakeScenario: + scenario_key: str + scenario_id: str + sub_goals: list[FakeSubGoal] + + def setup(self, world: Any) -> object: + del world + return None + + def ready(self, world: Any) -> object: + del world + return None + + +class FakeCallRunner: + """Uploads a transcript through the adapter BEFORE returning, and can optionally trip the + cancel signal the instant a named scenario starts (for the cancel-mid-run test).""" + + def __init__( + self, + adapter: he.OutboundAdapter, + *, + cancel_path: Path | None = None, + cancel_on_scenario: str | None = None, + cancel_reason: str = "user_canceled", + delay_seconds: float = 0.05, + ) -> None: + self._adapter = adapter + self._cancel_path = cancel_path + self._cancel_on_scenario = cancel_on_scenario + self._cancel_reason = cancel_reason + self._delay_seconds = delay_seconds + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + if self._cancel_path is not None and scenario.scenario_key == self._cancel_on_scenario: + self._cancel_path.write_text( + json.dumps({"reason": self._cancel_reason}), encoding="utf-8" + ) + await asyncio.sleep(self._delay_seconds) + transcript = json.dumps( + [{"speaker_role": "assistant", "content": f"hello from {scenario.scenario_key}"}] + ).encode("utf-8") + artifact_id = await self._adapter.upload_artifact( + transcript, kind=ob.ArtifactKind.TRANSCRIPT, scenario_key=scenario.scenario_key + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=(Call(name="tool", arguments={}, result="ok", ok=True, error="", refused=False, at=0.0),), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=artifact_id, recording_artifacts=(), + ) + + +def _rfc3339(value: datetime) -> str: + return ob.format_rfc3339_millis(value) + + +class FakeScenarioSource: + def __init__(self, scenarios: list[FakeScenario]) -> None: + self._scenarios = scenarios + + async def build( + self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, + world_factory: Any, bundle_dir: Path, + ) -> list[FakeScenario]: + # `bundle_dir` (p12: the scenario-source adapter's own seam) is unused by this in-memory + # fake -- accepted only because `run_job` now forwards it to every `ScenarioSource.build`, + # injected or not. + del job, bundle, pool, world_factory, bundle_dir + # `operation` mirrors what `register_with_platform` (scenario_source.py) really sends -- + # this fake's own `scenario_keys`/`scenario_ids` bodies are otherwise arbitrary (never + # parsed by `FakeTransport`, which only inspects `operation` to pick a response), kept only + # so `test_scenarios_channel_uses_bearer_auth_never_api_key` and friends see two distinct, + # non-empty POST bodies. + await asyncio.to_thread( + scenarios_client.provision, + {"operation": "provision", "scenario_keys": [s.scenario_key for s in self._scenarios]}, + ) + await asyncio.to_thread( + scenarios_client.begin, {"operation": "begin", "scenario_ids": {}} + ) + return self._scenarios + + +# ================================================================================================= +# Deps builder. +# ================================================================================================= + + +@dataclass +class Harness: + tmp: Path + work: Path + source: Path + output: Path + job_path: Path + transport: FakeTransport + provisioner: FakeProvisioner + deps: he.HostedEntrypointDeps + bundle_dir: Path + + +def _build_harness( + *, + scenarios: list[FakeScenario], + fence_after: int | None = None, + fence_on_url_substring: str | None = None, + fence_on_terminal_event: bool = False, + cancel_on_scenario: str | None = None, + cancel_reason: str = "user_canceled", + corrupt_bundle: Callable[[Path], None] | None = None, + instances: int = 1, + always_unhealthy: bool = False, + parallelism: int = 1, + build_output: dict[str, Any] | None = None, + artifacts: HarnessArtifactPolicy | None = None, + # p12: leaves `deps.scenario_source` at its real default (`NotWiredScenarioSource`) instead of + # the usual `FakeScenarioSource` -- for the scenario_source.py wiring tests, which need + # `run_job` to actually choose between the default and the real `BundleScenarioSource` itself. + use_default_scenario_source: bool = False, + # p12: swaps in `_write_bundle_with_scenario_files` for the scenario-source wiring tests, which + # need real scenario documents hashed into the manifest's `files[]` (§2e `bundle_file_unlisted` + # otherwise rejects them at preflight) -- every other caller keeps the plain `_write_bundle`. + bundle_writer: Callable[[Path], Any] = _write_bundle, +) -> Harness: + tmp = Path(tempfile.mkdtemp(prefix="p10-e2e-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + bundle_dir = work / he.DEFAULT_BUNDLE_DIR_NAME + source.mkdir(parents=True, exist_ok=True) + bundle_writer(bundle_dir) + if corrupt_bundle is not None: + corrupt_bundle(bundle_dir) + + job_path = tmp / "job.json" + _write_job(job_path, _job(parallelism=parallelism, artifacts=artifacts)) + + if build_output is not None: + # `write_build_output` (process_runtime.py) writes here; no test previously did, so + # the baseline_frozen/parallelism_degraded block ran only its + # empty-dict fallback in every prior test. + output.mkdir(parents=True, exist_ok=True) + (output / "build.json").write_text(json.dumps(build_output), encoding="utf-8") + + capabilities = _capabilities() + transport = FakeTransport( + fence_after=fence_after, fence_on_url_substring=fence_on_url_substring, + fence_on_terminal_event=fence_on_terminal_event, + ) + provisioner = FakeProvisioner(instances=instances, always_unhealthy=always_unhealthy) + + cancel_path = tmp / "cancel.json" + + holder: dict[str, he.OutboundAdapter] = {} + + def build_call_runner( + adapter: he.OutboundAdapter, context: he.CallRunnerContext + ) -> FakeCallRunner: + holder["adapter"] = adapter + holder["call_runner_context"] = context + return FakeCallRunner( + adapter, cancel_path=cancel_path, cancel_on_scenario=cancel_on_scenario, + cancel_reason=cancel_reason, + ) + + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + bundle_source=he.DefaultBundleSource(), + scenario_source=( + he.NotWiredScenarioSource() if use_default_scenario_source else FakeScenarioSource(scenarios) + ), + build_transport=lambda: transport, + build_provider=lambda: provisioner, + build_call_runner=build_call_runner, + build_world_factory=lambda work_directory: FakeWorldFactory(), + cancel_path=cancel_path, + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + flush_window_seconds=5.0, + ) + return Harness( + tmp=tmp, work=work, source=source, output=output, job_path=job_path, transport=transport, + provisioner=provisioner, deps=deps, bundle_dir=bundle_dir, + ) + + +def _run(harness: Harness) -> int: + return asyncio.run(he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps)) + + +def _build_adapter( + transport: FakeTransport, *, extra_secret_values: tuple[str, ...] = () +) -> he.OutboundAdapter: + """Adapter-only fixture for tests that drive `OutboundAdapter` directly (no `run_job`) -- + mirrors `test_redaction_end_to_end_secret_never_crosses_any_channel`'s own inline construction, + factored out for reuse across other adapter-level tests.""" + capabilities = _capabilities() + channel_state = ob.ChannelState() + retry_policy = ob.RetryPolicy() + tmp = Path(tempfile.mkdtemp(prefix="p10-adapter-")) + events_spool = ob.OutboundSpool(tmp / "spool", "events", sequenced=True) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + return he.OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=extra_secret_values, + ) + + +# ================================================================================================= +# Pure-logic unit tests. +# ================================================================================================= + + +def test_resolve_parallelism_reads_the_raw_value_without_clamping() -> None: + # `RuntimeRequirements.parallelism` now exists -- `resolve_parallelism` + # must return it RAW; clamping here would make `parallelism_out_of_range` unreachable. + assert he.resolve_parallelism(_job(parallelism=1)) == 1 + assert he.resolve_parallelism(_job(parallelism=8)) == 8 + # An out-of-range value is preflight's to reject (§2e.7), not this function's to launder -- + # `RuntimeRequirements.parallelism` itself only enforces `ge=1`, so a too-large W passes model + # validation and must still reach `resolve_parallelism` unclamped. + assert he.resolve_parallelism(_job(parallelism=99)) == 99 + + +def test_out_of_range_parallelism_is_rejected_by_preflight_not_clamped() -> None: + # Confirms an out-of-range W reaches a + # `parallelism_out_of_range` preflight rejection (§2e.7), never a silently clamped W=8 run. + async def scenario() -> None: + harness = _build_harness(scenarios=[], parallelism=20) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # rejected before any provision, like §2e. + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["failure"]["code"] == "parallelism_out_of_range" + assert payload["failure"]["domain"] == "environment" + assert payload["failure"]["stage"] == "validating_environment" + + asyncio.run(scenario()) + + +def test_job_secret_purposes_maps_alias_to_purpose() -> None: + job = _job() + assert he.job_secret_purposes(job) == {TARGET_PROVIDER_ALIAS: "target_provider"} + + +def test_peek_secret_values_reads_without_deleting(tmp_path_factory: Path | None = None) -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-secrets-")) + path = tmp / "secrets.json" + path.write_text(json.dumps({"LIVEKIT_API_KEY": "sk-super-secret"}), encoding="utf-8") + values = he.peek_secret_values(path) + assert values == ("sk-super-secret",) + assert path.exists() # non-destructive read -- the provisioner still owns load-and-delete. + + +def test_peek_secret_values_missing_file_is_empty() -> None: + assert he.peek_secret_values(Path("/nonexistent/does-not-exist.json")) == () + + +def test_peek_target_provider_secret_values_filters_by_purpose_and_keeps_the_alias() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p14-secrets-")) + path = tmp / "secrets.json" + path.write_text( + json.dumps( + { + "LIVEKIT_API_KEY": "lk-secret", + "GITHUB_INSTALLATION_TOKEN": "gh-secret", + } + ), + encoding="utf-8", + ) + values = he.peek_target_provider_secret_values( + path, + {"LIVEKIT_API_KEY": "target_provider", "GITHUB_INSTALLATION_TOKEN": "source_checkout"}, + ) + assert values == {"LIVEKIT_API_KEY": "lk-secret"} + assert path.exists() # non-destructive read, same timing contract as peek_secret_values. + + +def test_peek_target_provider_secret_values_missing_file_is_empty() -> None: + assert ( + he.peek_target_provider_secret_values( + Path("/nonexistent/does-not-exist.json"), {"LIVEKIT_API_KEY": "target_provider"} + ) + == {} + ) + + +def test_peek_target_provider_secret_values_drops_an_alias_with_no_purpose_entry() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p14-secrets-")) + path = tmp / "secrets.json" + path.write_text(json.dumps({"UNKNOWN_ALIAS": "value"}), encoding="utf-8") + assert he.peek_target_provider_secret_values(path, {}) == {} + + +# ================================================================================================= +# CallRunner wiring (p14): the extended build_call_runner(adapter, context) seam, and the +# NotWired-stays-the-fallback / real-CallRunnerImpl split on `agent.connector`. +# ================================================================================================= + + +def _call_runner_context( + *, job: HarnessJob | None = None, evidence_seam: Any = EvidenceSeam.HTTP_TOOL +) -> he.CallRunnerContext: + return he.CallRunnerContext( + job=job or _job(), + bundle_dir=Path("/nonexistent/bundle"), + work_directory=Path("/nonexistent/work"), + evidence_seam=evidence_seam, + target_provider_secret_values={}, + attempt_number=1, + ) + + +def test_default_build_call_runner_returns_notwired_for_a_non_livekit_connector() -> None: + # `_job()`'s own default is `connector="vapi"` -- out of this worker's mission, by design. + runner = he._default_build_call_runner(mock.Mock(), _call_runner_context(job=_job(connector="vapi"))) + assert isinstance(runner, he.NotWiredCallRunner) + + +def test_default_build_call_runner_returns_notwired_for_retell_and_auto_too() -> None: + for connector in ("retell", "auto"): + runner = he._default_build_call_runner( + mock.Mock(), _call_runner_context(job=_job(connector=connector)) + ) + assert isinstance(runner, he.NotWiredCallRunner) + + +def test_default_build_call_runner_returns_a_real_call_runner_impl_for_livekit() -> None: + runner = he._default_build_call_runner(mock.Mock(), _call_runner_context(job=_job(connector="livekit"))) + assert isinstance(runner, he.CallRunnerImpl) + + +def test_call_runner_context_is_threaded_with_real_job_bundle_secrets_and_evidence_seam() -> None: + """End-to-end through the real `run_job` wiring point (~line 1728): `secret_purposes = + job_secret_purposes(job)` runs, `deps.peek_target_provider_secret_values` captures the alias + map BEFORE `pool.start()` deletes `secrets.json`, and the resulting `CallRunnerContext` reaches + whatever factory `deps.build_call_runner` names -- verified by capturing it, not by asserting + on `CallRunnerImpl` internals (that belongs to test_call_runner.py). + + `SecretDeletingProvisioner` mirrors the REAL `ProcessRuntimeProvider`'s own lifetime rule + (process_runtime.py:3535-3544): `secrets.json` is deleted on the provisioner's FIRST + `provision()` call, which `pool.start()` awaits synchronously. Without this, `FakeProvisioner` + never touches the file at all, and a capture that happened AFTER `pool.start()` instead of + before would still see the intact secrets file and still pass -- the deletion is what makes + the capture's timing actually load-bearing here, not just documented in a comment.""" + + class SecretDeletingProvisioner(FakeProvisioner): + def __init__(self, *args: Any, secrets_path: Path, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._secrets_path = secrets_path + + async def provision(self, *args: Any, **kwargs: Any) -> list[EnvironmentRuntime]: + runtimes = await super().provision(*args, **kwargs) + self._secrets_path.unlink(missing_ok=True) + return runtimes + + harness = _build_harness(scenarios=[]) + harness.deps.secrets_path.parent.mkdir(parents=True, exist_ok=True) + harness.deps.secrets_path.write_text( + json.dumps({TARGET_PROVIDER_ALIAS: "lk-secret-value"}), encoding="utf-8" + ) + harness.deps.build_provider = lambda: SecretDeletingProvisioner( + instances=1, secrets_path=harness.deps.secrets_path + ) + + captured: dict[str, he.CallRunnerContext] = {} + + def build_call_runner(adapter: he.OutboundAdapter, context: he.CallRunnerContext) -> FakeCallRunner: + captured["context"] = context + return FakeCallRunner(adapter, cancel_path=harness.deps.cancel_path) + + harness.deps.build_call_runner = build_call_runner + code = _run(harness) + assert code == he.EXIT_OK + + context = captured["context"] + assert context.job.job_id == "job-1" + assert context.bundle_dir == harness.bundle_dir + # `_base_manifest_body()` (this file's own bundle fixture) declares `evidence_seam: "http_tool"`. + assert context.evidence_seam is EvidenceSeam.HTTP_TOOL + assert context.attempt_number == 1 + # The load-bearing assertion: the secrets file is genuinely gone by the time `provision()` + # (inside `pool.start()`) returns -- if the capture had happened AFTER `pool.start()` instead + # of before, this map would be empty, not the real alias -> value pair. + assert context.target_provider_secret_values == {TARGET_PROVIDER_ALIAS: "lk-secret-value"} + assert not harness.deps.secrets_path.exists() + + +def test_row_counts_for_capability_returns_the_matching_store() -> None: + build_output = {"stores": [{"capability": "database", "row_counts": {"riders": 3}}]} + assert he.row_counts_for_capability(build_output, "database") == {"riders": 3} + + +def test_row_counts_for_capability_raises_when_the_capability_is_absent() -> None: + build_output = {"stores": [{"capability": "other", "row_counts": {}}]} + try: + he.row_counts_for_capability(build_output, "database") + except he.WorldFactoryError: + pass + else: + raise AssertionError("expected WorldFactoryError") + + +def test_cancel_state_reads_reason_from_file() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-cancel-")) + path = tmp / "cancel.json" + state = he.CancelState(path) + assert state.requested() is False + path.write_text(json.dumps({"reason": "ttl_exceeded"}), encoding="utf-8") + assert state.requested() is True + assert state.reason() is ob.TerminalReason.TTL_EXCEEDED + + +def test_world_pool_serializes_concurrent_provider_calls_end_to_end() -> None: + # R7: `hosted_entrypoint.py` used to wrap every provider in `SerializingProvider` before + # `WorldPool` ever saw it -- removed now that `WorldPool`'s own `_provider_lock` covers + # provision/reset/healthy/close (mutation-verified in review: `healthy` rides the same + # non-reentrancy rule as the other three, per v1.12 §4.5b). This repoints the old + # wrapper-level test at the SAME guarantee, one level up: `pool.start()` (a `provision()` call) + # racing `pool._reconcile()` (a `provision()` THEN a `healthy()` call) against the bare, + # unwrapped `FakeProvisioner` this module now wires directly. + async def scenario() -> None: + fake = FakeProvisioner(instances=1) + work = Path(tempfile.mkdtemp(prefix="p10-pool-serial-")) + pool = WorldPool( + fake, bundle=None, source=work, bundle_dir=work, work_directory=work, instances=1, + ) + await asyncio.gather(pool.start(), pool._reconcile()) + # `overlaps` is populated OUT-OF-BAND by `FakeProvisioner._serialized()` -- an in-band + # `assert` there would fire inside `_reconcile`'s own `except Exception` (a reconcile must + # never crash the pool) and never reach this frame, silently passing a broken lock. + assert fake.overlaps == [] + assert fake.provision_calls >= 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_scenarios_client_provision_unwraps_the_result_envelope() -> None: + # p13: keyed response (`{"scenarios": [{"scenario_key", "scenario_id"}, ...]}`), matching the + # real platform's `_provision_response` (services/hosted_harness.py:487-501) -- never a + # position-ordered `scenario_ids` array. + capabilities = _capabilities() + transport = FakeTransport() + client = he.ScenariosClient(capabilities, transport) + result = client.provision( + { + "operation": "provision", "name": "run-1", + "personas": [{"scenario_key": "a"}, {"scenario_key": "b"}], + } + ) + assert result == { + "run_test_id": "run-test-1", + "scenarios": [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "b", "scenario_id": "platform-b"}, + ], + } + + +def test_scenarios_client_provision_and_begin_hit_the_same_single_url() -> None: + # p13: Azain's router mints exactly ONE url per attempt (views/hosted_harness.py:78-90's + # `scenarios` detail `@action`, no `url_path`; urls.py:128-132) -- `provision_path`/ + # `begin_path` default to an EMPTY suffix now (not the old guessed `"provision/"`/`"begin/"`, + # which 404 against the real router), so both calls land on `capabilities.endpoints.scenarios` + # itself, discriminated only by the body's `operation` field. + capabilities = _capabilities() + transport = FakeTransport() + client = he.ScenariosClient(capabilities, transport) + client.provision({"operation": "provision", "name": "run-1", "personas": []}) + client.begin({"operation": "begin", "run_test_id": "run-test-1", "scenario_keys": []}) + urls = [call["url"] for call in transport.calls] + assert urls == [capabilities.endpoints.scenarios, capabilities.endpoints.scenarios] + + +def test_scenarios_client_fencing_latches_the_shared_channel_state() -> None: + capabilities = _capabilities() + transport = FakeTransport(fence_after=1) + channel_state = ob.ChannelState() + client = he.ScenariosClient(capabilities, transport, channel_state=channel_state) + try: + client.provision({"operation": "provision", "name": "run-1", "personas": []}) + except ob.HostedFencedError: + pass + else: + raise AssertionError("expected HostedFencedError") + try: + channel_state.check() + except ob.HostedFencedError: + pass + else: + raise AssertionError("channel_state should now be latched for every other channel too") + + +# ================================================================================================= +# Targeted orchestration tests. +# ================================================================================================= + + +def test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event() -> None: + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-boot-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + source.mkdir(parents=True, exist_ok=True) + job_path = tmp / "job.json" + _write_job(job_path, _job()) + + def _raise() -> ob.HostedCapabilities: + raise ob.CapabilitiesError("capabilities_file_missing", "no file") + + deps = he.HostedEntrypointDeps(load_capabilities=_raise) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_BOOT_FAILURE + assert code != he.EXIT_FENCED + assert not (work / he.EVENTS_SPOOL_DIR_NAME).exists() # no channel was ever built. + + asyncio.run(scenario()) + + +def test_preflight_rejection_reaches_a_failed_terminal_event_before_any_provision() -> None: + async def scenario() -> None: + harness = _build_harness( + scenarios=[], + corrupt_bundle=lambda bundle_dir: (bundle_dir / "db" / "schema.sql").unlink(), + ) + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # "BEFORE any provision" + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + assert payload["failure"]["domain"] == "environment" + assert payload["failure"]["stage"] == "validating_environment" + assert payload["failure"]["code"] == "bundle_file_missing" + + asyncio.run(scenario()) + + +def test_hosted_fenced_error_stops_emitting_and_exits_3_with_no_terminal_event() -> None: + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + # Fenced mid-attempt (during the scenario's own receipt push), well after provisioning -- + # proves both halves of the contract: nothing further is ever emitted, AND close() still + # runs (it is unconditional after scheduler.run(), not gated on fencing). + harness = _build_harness(scenarios=scenarios, fence_on_url_substring="/results/") + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True # close() still runs on the way out. + + asyncio.run(scenario()) + + +def test_cancel_mid_run_synthesizes_a_skipped_receipt_for_the_unstarted_scenario() -> None: + async def scenario() -> None: + order: list[str] = [] + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + if ( + "/results/" in url and method == "POST" + and json_body is not None and json_body.get("status") == "skipped" + ): + order.append("skipped_receipt_delivered") + return response + + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + terminals = transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "user_canceled" + statuses = {key[1]: body["status"] for key, body in transport.receipts.items()} + assert statuses.get("first") == "passed" + assert statuses.get("second") == "skipped" + + # outbound-channels.md v1.3 Sequencing ("terminal event -> skipped receipts -> + # manifest") -- the skipped receipt for "second" must reach the platform, and strictly + # after the terminal event, never before or instead of it. + assert order == ["terminal_delivered", "skipped_receipt_delivered"] + + # the "skipped" receipt body (exact) per outbound-channels.md's own six-field list -- + # not just its `status`. + skipped_body = transport.receipts[("job-1", "second")] + assert skipped_body["scenario_attempt"] == 1 + assert skipped_body["world_index"] is None + assert skipped_body["sub_goals"] == [] + assert skipped_body["evaluations"] == [] + assert skipped_body["call"] is None + assert skipped_body["failure"] is None + + # a CANCELED run is cut short -- the manifest must say so. + assert transport.manifests[-1]["complete"] is False + + asyncio.run(scenario()) + + +def test_fenced_run_result_emits_zero_skipped_receipts() -> None: + # `RunResult.fenced` must gate the entrypoint's own call to + # `emit_skipped_receipts` -- checked here at the call site itself, not left to the scheduler's + # own internal no-op. `HostedScheduler.run` is monkeypatched to hand back a fenced `RunResult` + # regardless of how the real run went, since the real `OutboundAdapter` has no path that lets + # a channel fence reach `WorldPool.fenced` (it swallows `HostedFencedError` internally -- + # see `OutboundAdapter._guarded`), so this is the only reliable way to exercise the branch + # end-to-end through `run_job`. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, instances=1) + + fence_exc = ob.HostedFencedError( + ob.ChannelError(ob.ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + original_run = HostedScheduler.run + original_emit = HostedScheduler.emit_skipped_receipts + emit_calls: list[RunResult] = [] + + async def fenced_run(self: HostedScheduler, scns: Any) -> RunResult: + real = await original_run(self, scns) + return RunResult(receipts=real.receipts, aborted=real.aborted, fenced=fence_exc) + + async def counting_emit(self: HostedScheduler, result: RunResult) -> None: + emit_calls.append(result) + await original_emit(self, result) + + HostedScheduler.run = fenced_run # type: ignore[method-assign] + HostedScheduler.emit_skipped_receipts = counting_emit # type: ignore[method-assign] + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.run = original_run + HostedScheduler.emit_skipped_receipts = original_emit + + assert code == he.EXIT_OK + assert emit_calls == [] + + asyncio.run(scenario()) + + +def test_emit_skipped_receipts_failure_does_not_lose_the_terminal_or_the_exit_code() -> None: + # a failure inside the post-terminal `scheduler.emit_skipped_receipts` call must be + # swallowed locally (matching every other best-effort post-terminal emission in this module), + # never mask the terminal already delivered or flip the exit code. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + + original_emit = HostedScheduler.emit_skipped_receipts + + async def poisoned_emit(self: HostedScheduler, result: RunResult) -> None: + raise RuntimeError("synthetic emit_skipped_receipts failure") + + HostedScheduler.emit_skipped_receipts = poisoned_emit # type: ignore[method-assign] + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.emit_skipped_receipts = original_emit + + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "user_canceled" + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("first") == "passed" + + asyncio.run(scenario()) + + +def test_cancel_mid_run_with_ttl_exceeded_reports_that_reason() -> None: + # the exit-code table's CANCELED branch is only ever exercised with `user_canceled` + # elsewhere in this file -- `ttl_exceeded` is the contract's other legal reason + # (`ob.TerminalReason`) and goes through the exact same `CancelState.reason()` path. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness( + scenarios=scenarios, cancel_on_scenario="first", cancel_reason="ttl_exceeded", + instances=1, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "ttl_exceeded" + + asyncio.run(scenario()) + + +def test_malformed_job_json_exits_crashed_with_no_terminal_event() -> None: + # EXIT_CRASHED -- a malformed job.json exits non-zero with no channel of its own + # kind (capabilities load already succeeded, but nothing typed can describe the failure). + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-badjob-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + source.mkdir(parents=True, exist_ok=True) + job_path = tmp / "job.json" + job_path.write_text("{this is not valid json", encoding="utf-8") + + capabilities = _capabilities() + transport = FakeTransport() + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + build_transport=lambda: transport, + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + ) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_CRASHED + assert code != he.EXIT_FENCED + assert transport.terminal_events() == [] + + asyncio.run(scenario()) + + +def test_world_pool_exhaustion_reaches_a_failed_terminal_world_pool_exhausted() -> None: + # `RunResult.aborted` -> a terminal FAILED with domain `infrastructure`, stage + # `running`, code `world_pool_exhausted`. Driven for real: the single world never passes its + # `healthy()` probe, so `WorldPool.lease()` exhausts its reconcile budget and raises + # `NoWorldsAvailable`, which `HostedScheduler.run()` turns into `RunResult.aborted`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, always_unhealthy=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + assert payload["failure"]["domain"] == "infrastructure" + assert payload["failure"]["stage"] == "running" + assert payload["failure"]["code"] == "world_pool_exhausted" + assert harness.provisioner.closed is True + # The aborted lane still finalizes like a completed one: nothing further will ever + # upload, so the manifest must say complete AND carry the three kinds the platform + # refuses to ack a complete manifest without. + assert harness.transport.manifests, "no artifact manifest was pushed" + final_manifest = harness.transport.manifests[-1] + assert final_manifest["complete"] is True + kinds = {entry["kind"] for entry in final_manifest["entries"]} + assert {"build", "result", "log"} <= kinds + + asyncio.run(scenario()) + + +def test_fence_landing_on_the_final_drain_still_exits_fenced() -> None: + # a fence that 403s specifically the events + # POST carrying the terminal event (not an earlier one) must still exit 3 with no terminal + # event DELIVERED, never a stale pre-drain fence check that misses it. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, fence_on_terminal_event=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True + + asyncio.run(scenario()) + + +def test_fence_from_scenarios_client_exits_fenced_not_crashed() -> None: + # `ScenariosClient._post` re-raises `HostedFencedError` after latching -- + # this must reach `run_job`'s typed handler around `scenario_source.build()`, not fall through + # to a bare `except Exception` (exit 1, and the world pool leaked). + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1, fence_on_url_substring="/scenarios/") + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + assert harness.transport.terminal_events() == [] + assert harness.provisioner.closed is True # the pool must not leak on this path either. + + asyncio.run(scenario()) + + +def test_scenarios_channel_uses_bearer_auth_never_api_key() -> None: + # outbound-channels.md calls out bearer + X-Harness-Fence by name for pre-allocation -- + # never `X-Api-Key`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + scenarios_calls = [call for call in harness.transport.calls if "/scenarios/" in call["url"]] + assert scenarios_calls, "expected at least one call against endpoints.scenarios" + for call in scenarios_calls: + assert call["headers"].get("Authorization", "").startswith("Bearer ") + assert "X-Api-Key" not in call["headers"] + assert "X-Harness-Fence" in call["headers"] + + asyncio.run(scenario()) + + +def test_process_world_factory_raises_when_stores_recorded_but_no_endpoint() -> None: + # The endpoint-lost guard: build.json records seeded stores but the runtime has no + # postgres endpoint -- a provisioner fault that must fail loudly, never silently degrade + # to a storeless world. build.json is present and readable so the guard, not the + # "build.json unreadable" path, is what fires. + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-endpoint-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": [{"capability": "database", "row_counts": {}}]}), + encoding="utf-8", + ) + factory = he.ProcessWorldFactory(tmp) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + try: + await factory.create(runtime, rng=random.Random(0)) + except he.WorldFactoryError: + pass + else: + raise AssertionError( + "expected WorldFactoryError when build.json records stores but no endpoint exists" + ) + + asyncio.run(scenario()) + + +def test_process_world_factory_builds_storeless_world_when_no_endpoint_and_no_stores() -> None: + # The relaxed storeless path: no postgres endpoint AND build.json records no stores is a + # legal stateless bundle -- the factory hands back a StorelessWorld, not an error. + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-storeless-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": []}), encoding="utf-8" + ) + factory = he.ProcessWorldFactory(tmp) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + world = await factory.create(runtime, rng=random.Random(0)) + assert isinstance(world, he.StorelessWorld) + + asyncio.run(scenario()) + + +def test_process_world_factory_raises_when_build_json_has_no_matching_store() -> None: + async def scenario() -> None: + tmp = Path(tempfile.mkdtemp(prefix="p10-wf-store-")) + (tmp / "artifacts").mkdir(parents=True, exist_ok=True) + (tmp / "artifacts" / "build.json").write_text( + json.dumps({"stores": [{"capability": "other", "row_counts": {}}]}), encoding="utf-8" + ) + factory = he.ProcessWorldFactory(tmp) + endpoint = RuntimeEndpoint( + capability="database", protocol="postgres", address="postgresql://u:p@localhost/db", + ) + runtime = EnvironmentRuntime( + runtime_id="digest:w0", world_index=0, bundle_digest="digest", + state=RuntimeState.READY, endpoints={"database": endpoint}, + ) + try: + await factory.create(runtime, rng=random.Random(0)) + except he.WorldFactoryError: + pass + else: + raise AssertionError("expected WorldFactoryError for a build.json with no matching store") + + asyncio.run(scenario()) + + +def test_build_json_two_stores_emit_two_baseline_frozen_events() -> None: + # Each store in build.json's stores list gets its own baseline_frozen event, not just the first. + async def scenario() -> None: + build_output = { + "stores": [ + { + "capability": "database", "baseline_reference": "ref-database", + "inputs_digest": "digest-database", + }, + { + "capability": "cache", "baseline_reference": "ref-cache", + "inputs_digest": "digest-cache", + }, + ], + } + harness = _build_harness(scenarios=[], instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + baseline_events = [ + record for record in harness.transport.event_records + if record.get("type") == "baseline_frozen" + ] + assert len(baseline_events) == 2 + refs = {event["payload"]["baseline_ref"] for event in baseline_events} + assert refs == {"ref-database", "ref-cache"} + + asyncio.run(scenario()) + + +def test_build_json_degrade_payload_matches_the_recorded_values() -> None: + # `parallelism_degraded`'s payload must mirror build.json's own requested/effective/reason values exactly. + async def scenario() -> None: + build_output = { + "requested_parallelism": 2, "effective_parallelism": 1, + "degrade_reason": "conformance_gate_failed", + } + harness = _build_harness(scenarios=[], instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + degrade_events = [ + record for record in harness.transport.event_records + if record.get("type") == "parallelism_degraded" + ] + assert len(degrade_events) == 1 + payload = degrade_events[0]["payload"] + assert payload == {"requested": 2, "effective": 1, "reason": "conformance_gate_failed"} + + asyncio.run(scenario()) + + +def test_build_json_fixed_port_at_w1_does_not_crash() -> None: + # `requested == effective == 1` with + # `degrade_reason: fixed_port` is not representable as a `parallelism_degraded` event + # (`1 <= effective < requested` fails); this must degrade to a `log`, never crash the run. + async def scenario() -> None: + build_output = { + "requested_parallelism": 1, "effective_parallelism": 1, "degrade_reason": "fixed_port", + } + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1, build_output=build_output) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.closed is True # the pool must not be orphaned by a crash. + degrade_events = [ + record for record in harness.transport.event_records + if record.get("type") == "parallelism_degraded" + ] + assert degrade_events == [] # not representable -- a log event carries it instead. + log_events = [ + record for record in harness.transport.event_records if record.get("type") == "log" + ] + assert any("fixed_port" in record["payload"]["message"] for record in log_events) + # a substring shared with the pydantic error text the blanket `except Exception` + # would ALSO produce if the `effective < requested` guard were reverted -- this is the one + # phrase that only the guard's own `else` branch ever writes, so it is what actually tells + # the guard apart from the catch-all swallowing a ValidationError. + assert any( + "no parallelism_degraded event is representable" in record["payload"]["message"] + for record in log_events + ) + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + +def test_e2e_two_scenarios_one_pass_one_fail_reaches_completed_and_exits_0() -> None: + async def scenario() -> None: + scenarios = [ + FakeScenario("passing", "platform-passing", [FakeSubGoal("holds", True)]), + FakeScenario("failing", "platform-failing", [FakeSubGoal("holds", False)]), + ] + harness = _build_harness(scenarios=scenarios, instances=1) + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["reason"] is None + assert payload["failure"] is None + assert payload["scenario_counts"] == {"passed": 1, "failed": 1, "errored": 0, "skipped": 0} + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed", "failing": "failed"} + + # both receipts reference an already-uploaded, already-acked transcript artifact. + for key, body in harness.transport.receipts.items(): + del key + transcript = body["call"]["transcript_artifact"] + assert transcript is not None + assert transcript.split(":", 1)[1] in harness.transport.artifacts + + assert len(harness.transport.manifests) >= 1 + assert harness.transport.manifests[-1]["complete"] is True + # Channel 3's level table: a complete manifest carries the run-level kinds + # (`build`, `result`, `log`, plus the event trace) alongside the transcripts -- + # the platform refuses the manifest ack without them. + manifest_kinds = [e["kind"] for e in harness.transport.manifests[-1]["entries"]] + assert manifest_kinds.count("transcript") == 2 + for required in ("build", "result", "log"): + assert required in manifest_kinds, manifest_kinds + + assert harness.provisioner.provision_calls >= 1 + assert harness.provisioner.closed is True + + # Scenario pre-allocation (item 3) actually ran against endpoints.scenarios -- p13: a + # single url, discriminated by the body's `operation` field (never a `/provision/`/ + # `/begin/` url suffix, which 404s against the real platform router). + assert any( + body.get("operation") == "provision" for _, body in harness.transport.scenarios_calls + ) + assert any( + body.get("operation") == "begin" for _, body in harness.transport.scenarios_calls + ) + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests closing mutation-honesty gaps -- cases where mutating the guarded code paths +# would have passed the suite unnoticed. +# ================================================================================================= + + +def test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped() -> None: + # deleting the top-level `finally: pool.close()` backstop passed 29/29 -- every prior + # test's exception path had an explicit close ahead of it. `MemoryError` matches none of + # `run_job`'s typed handlers around `scenario_source.build()`, so it propagates straight past + # the `finally` with no explicit close anywhere on this path -- only the backstop can close it. + async def scenario() -> None: + class ExplodingScenarioSource: + async def build(self, job, bundle, scenarios_client, *, pool, world_factory, bundle_dir): + del job, bundle, scenarios_client, pool, world_factory, bundle_dir + raise MemoryError("boom") + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = ExplodingScenarioSource() + raised = False + try: + await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + except MemoryError: + raised = True + assert raised, "expected the untyped exception to propagate past run_job" + assert harness.provisioner.closed is True # only the finally backstop could have done this + + asyncio.run(scenario()) + + +def test_process_runtime_error_uses_the_carried_domain_over_the_fallback_map() -> None: + # v1.15 §2f: the producer resolves and carries `domain` at the raise site -- `spawn_failed`'s + # managed/source split is no longer re-derived from a manifest lookup here (mutation: force + # `_process_runtime_error_domain` back to ignoring `exc.domain` and this test catches it, since + # the SAME code with two different carried domains would then collapse to one fallback value). + async def run_case(code: str, domain: FailureDomain | None, expected_domain: str) -> None: + class RaisingProvisioner(FakeProvisioner): + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract, instances + raise ProcessRuntimeError("build", code, "synthetic failure", domain=domain) + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.build_provider = lambda: RaisingProvisioner(instances=1) + result = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert result == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == expected_domain + assert failure["stage"] == "building_environment" + + # The same code, two different CARRIED domains -- proves the domain is read off the + # exception, not re-derived from `code`/`process` (the map alone could never distinguish + # these two cases, since both are `spawn_failed`). + asyncio.run(run_case("spawn_failed", FailureDomain.AGENT, "agent")) # source, carried + asyncio.run(run_case("spawn_failed", FailureDomain.INFRASTRUCTURE, "infrastructure")) # managed, carried + # No carried domain (as a raise site outside this module's control might produce) -- the + # closed map is consulted as a fallback only. + asyncio.run(run_case("build_failed", None, "agent")) + asyncio.run(run_case("seed_failed", None, "environment")) + + +def test_scenario_entry_missing_scenario_key_fails_cleanly_never_an_attributeerror() -> None: + # karthik-integration-changes.md K1: the Scenario Generation Contract's own model may not + # carry `scenario_key` yet, and `hosted_scheduler.py` reads `scenario.scenario_key` with plain + # attribute access -- an entry that lacks the field entirely used to raise AttributeError deep + # in the scheduler (no terminal event, a crash exit code) instead of failing the job cleanly. + class ScenarioMissingKey: + scenario_id = "id-x" + sub_goals: list[Any] = [] + + def setup(self, world: Any) -> object: + del world + return None + + def ready(self, world: Any) -> object: + del world + return None + + class RawScenarioSource: + """Returns entries verbatim -- unlike `FakeScenarioSource`, never reads `.scenario_key` + itself before handing them back, so the entrypoint's own validation is what is under + test, not this fixture crashing first.""" + + def __init__(self, scenarios: list[Any]) -> None: + self._scenarios = scenarios + + async def build( + self, job: Any, bundle: Any, scenarios_client: he.ScenariosClient, *, pool: Any, + world_factory: Any, bundle_dir: Path, + ) -> list[Any]: + del job, bundle, scenarios_client, pool, world_factory, bundle_dir + return self._scenarios + + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = RawScenarioSource([ScenarioMissingKey()]) + result = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert result == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "environment" + assert "scenario_key" in failure["message"] + + asyncio.run(scenario()) + + +def test_finish_emits_the_terminal_before_closing_the_pool() -> None: + # moving `_bounded_close()` before `emit_terminal()` inside `_finish` passed 29/29 -- + # nothing recorded the RELATIVE order of the two. This records both against one shared timeline. + async def scenario() -> None: + order: list[str] = [] + tmp = Path(tempfile.mkdtemp(prefix="p10-order-")) + work = tmp / "work" + source = work / "source" + output = work / "artifacts" + bundle_dir = work / he.DEFAULT_BUNDLE_DIR_NAME + source.mkdir(parents=True, exist_ok=True) + _write_bundle(bundle_dir) + job_path = tmp / "job.json" + _write_job(job_path, _job(parallelism=1)) + capabilities = _capabilities() + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + return response + + class OrderTrackingProvisioner(FakeProvisioner): + async def close(self, *, work_directory: Path) -> None: + await super().close(work_directory=work_directory) + order.append("pool_closed") + + transport = OrderTrackingTransport() + provisioner = OrderTrackingProvisioner(instances=1) + deps = he.HostedEntrypointDeps( + load_capabilities=lambda: capabilities, + bundle_source=he.DefaultBundleSource(), + scenario_source=FakeScenarioSource([]), + build_transport=lambda: transport, + build_provider=lambda: provisioner, + build_world_factory=lambda work_directory: FakeWorldFactory(), + cancel_path=tmp / "cancel.json", + secrets_path=tmp / "secrets.json", + install_sigterm_handler=lambda cancel_state: (lambda: None), + flush_window_seconds=5.0, + ) + code = await he.run_job(job_path, source, output, deps=deps) + assert code == he.EXIT_OK + assert order == ["terminal_delivered", "pool_closed"] + + asyncio.run(scenario()) + + +def test_redaction_end_to_end_secret_never_crosses_any_channel() -> None: + # dropping the `extra_secret_values` threading (event_builder AND build_result_receipt), + # or the two pre-existing `redact_outbound_text` calls (world_unhealthy.cause, + # receipt.failure.message), all passed 29/29 -- nothing pinned redaction on the wire. Drives the + # secret through a log message, world_unhealthy, a terminal failure, and a receipt failure -- + # the free-text fields the contract names -- and reads what actually reached the transport. + async def scenario() -> None: + secret = "sk-live-eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + capabilities = _capabilities() + transport = FakeTransport() + channel_state = ob.ChannelState() + retry_policy = ob.RetryPolicy() + tmp = Path(tempfile.mkdtemp(prefix="p10-redact-")) + events_spool = ob.OutboundSpool(tmp / "spool", "events", sequenced=True) + events_client = ob.EventsClient( + capabilities, events_spool, transport, retry_policy=retry_policy, + channel_state=channel_state, + ) + results_client = ob.ResultsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + artifacts_client = ob.ArtifactsClient( + capabilities, transport, retry_policy=retry_policy, channel_state=channel_state, + ) + adapter = he.OutboundAdapter( + capabilities, + events_spool=events_spool, + events_client=events_client, + results_client=results_client, + artifacts_client=artifacts_client, + channel_state=channel_state, + extra_secret_values=(secret,), + ) + + await adapter.log(level="error", message=f"boom: {secret}") + await adapter.world_unhealthy(world_index=0, cause=f"probe failed: {secret}") + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", + message=f"failed calling out with {secret}", + ), + ) + ) + await adapter.emit_terminal( + stage=HarnessStage.FAILED, + failure={ + "domain": "infrastructure", "stage": "building_environment", + "code": "provision_failed", "message": f"connection failed: {secret}", + }, + ) + await adapter.drain(complete=True) + + for record in transport.event_records: + assert secret not in json.dumps(record) + for body in transport.receipts.values(): + assert secret not in json.dumps(body) + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any("***" in r["payload"]["message"] for r in log_events) + world_unhealthy_events = [ + r for r in transport.event_records if r.get("type") == "world_unhealthy" + ] + assert any("***" in r["payload"]["cause"] for r in world_unhealthy_events) + terminal = transport.terminal_events()[0] + assert "***" in terminal["payload"]["failure"]["message"] + receipt_body = transport.receipts[("job-1", "s1")] + assert "***" in receipt_body["failure"]["message"] + + asyncio.run(scenario()) + + +def test_drain_loops_past_a_backlog_larger_than_one_batch_and_still_delivers_the_terminal() -> None: + # A backlog bigger + # than one `EVENTS_MAX_BATCH` (100) previously stranded the terminal event, the highest + # sequence, while still exiting 0. A call runner that logs 260 chatter events before returning + # reproduces the same shape end to end. + async def scenario() -> None: + class ChattyCallRunner: + def __init__(self, adapter: he.OutboundAdapter, *, log_count: int) -> None: + self._adapter = adapter + self._log_count = log_count + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + for i in range(self._log_count): + await self._adapter.log(level="info", message=f"chatter {i}") + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.build_call_runner = lambda adapter, context: ChattyCallRunner(adapter, log_count=260) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # must not be stranded behind the backlog + + chatter_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and "chatter" in record["payload"].get("message", "") + ] + assert len(chatter_logs) == 260 # the WHOLE backlog drained, not just the first batch + + asyncio.run(scenario()) + + +def test_call_aborted_with_no_ended_at_still_produces_a_receipt() -> None: + # `CallAborted.partial.ended_at` is legitimately `None` (the + # call started but never finished). Before the fix, `build_result_receipt` raised inside + # `OutboundAdapter.receipt()` (outbound.CallSummary.ended_at is a required str), swallowed by + # `HostedScheduler._emit`'s blanket `except Exception` -- the scenario reached the wire with NO + # receipt at all despite `terminal.scenario_counts` claiming one `errored`. + async def scenario() -> None: + class AbortingCallRunner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + now = _rfc3339(datetime.now(timezone.utc)) + raise CallAborted( + "ran out of time before the call finished", + partial=CallOutcome( + calls=(), turns=1, started_at=now, ended_at=None, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.build_call_runner = lambda adapter, context: AbortingCallRunner() + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("s1") == "errored" + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["call"] is not None + assert receipt_body["call"]["ended_at"] == receipt_body["call"]["started_at"] + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["scenario_counts"]["errored"] == 1 + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests: artifact-level admission coverage, terminal-delivery/receipt-rejection/ +# message-capping edge cases, and mutation-survivor gaps around cancellation and the CANCELED manifest. +# ================================================================================================= + + +def test_metadata_only_artifact_level_refuses_transcript_upload_end_to_end() -> None: + # `_ARTIFACT_LEVEL_FORBIDDEN_KINDS` had zero suite coverage -- emptying the table passed + # every test. Drives a real `metadata-only` job through the default transcript-uploading call + # runner and reads what actually reached the transport. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, + artifacts=HarnessArtifactPolicy(level=ArtifactLevel.METADATA_ONLY), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + # metadata-only forbids transcripts, but the level table MANDATES the run-level + # kinds -- so exactly build/result/log reach the transport, and nothing scenario-scoped. + entries = harness.transport.manifests[-1]["entries"] + uploaded_kinds = sorted(entry["kind"] for entry in entries) + assert "transcript" not in uploaded_kinds + for required in ("build", "result", "log"): + assert required in uploaded_kinds, uploaded_kinds + listed_digests = {e["artifact_id"].split(":", 1)[1] for e in entries} + assert set(harness.transport.artifacts) == listed_digests # no unlisted bytes + + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["status"] == "passed" # a refused upload must not error the scenario + assert receipt_body["call"]["transcript_artifact"] is None + + log_events = [r for r in harness.transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "kind=transcript" in r["payload"]["message"] + and "forbidden at level=metadata-only" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_traces_artifact_level_refuses_recording_upload_end_to_end() -> None: + # A second shape at a different level -- `traces` allows transcripts but forbids + # recordings. A custom call runner uploads both so the table's per-kind behaviour is visible, + # not just its per-level all-or-nothing behaviour. + async def scenario() -> None: + class RecordingCallRunner: + def __init__(self, adapter: he.OutboundAdapter) -> None: + self._adapter = adapter + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + transcript_id = await self._adapter.upload_artifact( + b"transcript-bytes", kind=ob.ArtifactKind.TRANSCRIPT, + scenario_key=scenario.scenario_key, + ) + recording_id = await self._adapter.upload_artifact( + b"recording-bytes", kind=ob.ArtifactKind.RECORDING_COMBINED, + scenario_key=scenario.scenario_key, + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=transcript_id, + recording_artifacts=(recording_id,) if recording_id else (), + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, + artifacts=HarnessArtifactPolicy(level=ArtifactLevel.TRACES), + ) + harness.deps.build_call_runner = lambda adapter, context: RecordingCallRunner(adapter) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + receipt_body = harness.transport.receipts[("job-1", "s1")] + assert receipt_body["status"] == "passed" + assert receipt_body["call"]["transcript_artifact"] is not None # traces allows transcripts + assert receipt_body["call"]["recording_artifacts"] == [] # recordings refused at traces + + transcript_digest = receipt_body["call"]["transcript_artifact"].split(":", 1)[1] + assert transcript_digest in harness.transport.artifacts + uploaded_kinds = {e["kind"] for e in harness.transport.manifests[-1]["entries"]} + assert "recording_combined" not in uploaded_kinds # the recording never uploaded + assert {"build", "result", "log"} <= uploaded_kinds + recording_digest = hashlib.sha256(b"recording-bytes").hexdigest() + assert recording_digest not in harness.transport.artifacts # not even the bytes + + log_events = [r for r in harness.transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "kind=recording_combined" in r["payload"]["message"] + and "forbidden at level=traces" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_secrets_unlink_failure_after_the_terminal_does_not_lose_the_terminal_event() -> None: + # A non-writable secrets directory must not cost the terminal event -- the + # unlink now runs AFTER `emit_terminal` and is wrapped, so an OSError there is logged, not + # raised past `run_job`. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + guard_dir = Path(tempfile.mkdtemp(prefix="p10-ro-")) + secrets_path = guard_dir / "secrets.json" + secrets_path.write_text('{"A": "x"}', encoding="utf-8") + os.chmod(guard_dir, stat.S_IRUSR | stat.S_IXUSR) # read-only directory -> unlink raises + harness.deps.secrets_path = secrets_path + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + finally: + os.chmod(guard_dir, stat.S_IRWXU) + + asyncio.run(scenario()) + + +def test_events_channel_dying_on_the_final_drain_exits_terminal_undelivered() -> None: + # The events channel dies specifically on the flush that carries the terminal + # (sustained 5xx exhausts the retry budget) -- exit 0 would falsely claim a flush that never + # happened. `terminal_undelivered` catches this via the spool watermark never reaching the + # terminal's own sequence. + async def scenario() -> None: + class DyingOnTerminalTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if ( + method == "POST" and "/events/" in url + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + return ob.TransportResponse( + 500, {"error": "server_error", "message": "boom", "retryable": True}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = DyingOnTerminalTransport() + harness.deps.build_transport = lambda: transport + harness.deps.retry_policy = lambda: ob.RetryPolicy( + initial_backoff_seconds=0.0, max_backoff_seconds=0.0, max_attempts=3, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_TERMINAL_UNDELIVERED + assert transport.terminal_events() == [] # never actually reached the platform + + asyncio.run(scenario()) + + +def test_platform_rejecting_the_terminal_event_exits_terminal_undelivered() -> None: + # A different shape: the platform permanently rejects the terminal item itself (a per-item rejection + # inside an otherwise-200 response), distinct from a dead channel. + async def scenario() -> None: + class RejectingTerminalTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if method == "POST" and "/events/" in url and isinstance(data, (bytes, bytearray)): + body = json.loads(bytes(data).decode("utf-8")) + events = body.get("events", []) + terminal = [e for e in events if e.get("type") == "terminal"] + if terminal: + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + keep = [e for e in events if e.get("type") != "terminal"] + self.event_records.extend(keep) + rejected = [ + {"sequence": e["sequence"], "code": "payload_invalid", "message": "nope"} + for e in terminal + ] + watermark = max((e["sequence"] for e in events), default=0) + return ob.TransportResponse( + 200, {"acked_through_sequence": watermark, "rejected": rejected}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = RejectingTerminalTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_TERMINAL_UNDELIVERED + assert transport.terminal_events() == [] # rejected, never landed as a delivered record + + asyncio.run(scenario()) + + +def test_receipt_rejection_by_the_platform_is_logged() -> None: + # `ResultsClient.push()` returns `ReceiptPushResult(error=...)` on a permanent rejection + # rather than raising -- nothing inspected the return value before this fix, so the contract's + # "guest logs, no retry" obligation for e.g. 409 receipt_conflict went unmet. + async def scenario() -> None: + class RejectingResultsTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if method == "POST" and "/results/" in url and json_body is not None: + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + return ob.TransportResponse( + 409, + {"error": "receipt_conflict", "message": "digest mismatch", "retryable": False}, + {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + transport = RejectingResultsTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" + and "s1" in r["payload"]["message"] + and "receipt_conflict" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_receipt_failure_message_is_capped_like_the_terminal_message() -> None: + # `receipt().failure.message` was uncapped while `emit_terminal` caps at 4KB -- an + # oversized receipt failure message is the most plausible route into a 413 (the same silent-loss + # shape as a rejected receipt). Adapter-level: no `run_job` needed for a pure formatting check. + async def scenario() -> None: + transport = FakeTransport() + adapter = _build_adapter(transport) + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", message="y" * 20_000, + ), + ) + ) + body = transport.receipts[("job-1", "s1")] + message = body["failure"]["message"] + assert len(message) <= he._TERMINAL_FAILURE_MESSAGE_MAX_CHARS + assert message.endswith("…[truncated]") + + asyncio.run(scenario()) + + +def test_stored_result_artifact_redacts_a_secret_spanning_the_cap_boundary() -> None: + # `_upload_terminal_artifacts._clean` must redact BEFORE capping -- capping first can cut a + # secret in half at the 4KB boundary, and exact-substring redaction can no longer find the + # surviving fragment, making the stored `result` artifact the one outbound copy that leaks. + async def scenario() -> None: + secret = "sk-" + "S" * 33 # 36 chars, spanning the 4084-char truncation point + transport = FakeTransport() + adapter = _build_adapter(transport, extra_secret_values=(secret,)) + work = Path(tempfile.mkdtemp(prefix="p10-redact-cap-")) + receipt = ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="errored", sub_goals=(), evaluations=(), call=None, + failure=ReceiptFailure( + domain="agent", stage="running", code="call_failed", + message="x" * 4070 + secret, + ), + ) + await he._upload_terminal_artifacts( + adapter, work, RunResult(receipts=(receipt,), aborted=None) + ) + result_blobs = [b for b in transport.artifacts.values() if b"harness-result-set" in b] + assert result_blobs + blob = result_blobs[0] + fragments = [ + secret[i : i + 8] for i in range(len(secret) - 7) if secret[i : i + 8].encode() in blob + ] + assert not fragments, f"secret fragment survived into the stored result artifact: {fragments[:3]}" + + asyncio.run(scenario()) + + +def test_cancel_before_provision_skips_provisioning_entirely() -> None: + # The pre-provision cancel checkpoint (`hosted_entrypoint.py:1342` area) had no + # test driving a cancel signal written BEFORE `run_job` starts -- disabling all three + # checkpoints still passed the whole suite. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) + harness.deps.cancel_path.write_text( + json.dumps({"reason": "ttl_exceeded"}), encoding="utf-8" + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert harness.provisioner.provision_calls == 0 # canceled before provisioning ever ran + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + assert terminals[0]["payload"]["reason"] == "ttl_exceeded" + + # a CANCELED run is cut short -- the manifest must say so. + assert harness.transport.manifests[-1]["complete"] is False + + asyncio.run(scenario()) + + +def test_evaluation_wire_coerces_an_int_score_to_float() -> None: + # `_evaluation_wire`'s `float(score)` coercion had no direct unit test -- an + # int score digest-mismatches `build_result_receipt`'s own re-derivation and silently drops the + # whole receipt. + class IntScoreEvaluation: + kind = "metric" + name = "accuracy" + score = 1 + reason = "int not float" + + wire = he._evaluation_wire(IntScoreEvaluation()) + assert wire["score"] == 1.0 + assert isinstance(wire["score"], float) + + +def test_receipt_nulls_an_unacked_transcript_artifact_and_logs_it() -> None: + # A receipt naming an artifact id this adapter never uploaded (and therefore + # never acked) must ship `null`/`[]` on the wire, not the un-acked id -- the platform would 422 + # (`artifact_unknown`) the whole receipt otherwise. + async def scenario() -> None: + transport = FakeTransport() + adapter = _build_adapter(transport) + + class UnackedCall: + started_at = "2026-01-01T00:00:00.000Z" + ended_at = "2026-01-01T00:00:01.000Z" + duration_ms = 1000 + turns = 1 + transcript_artifact = "sha256:" + "a" * 64 # never uploaded through this adapter + recording_artifacts = ("sha256:" + "b" * 64,) + + await adapter.receipt( + ResultReceipt( + scenario_key="s1", scenario_id="platform-s1", scenario_attempt=1, world_index=0, + status="passed", sub_goals=(), evaluations=(), call=UnackedCall(), failure=None, + ) + ) + body = transport.receipts[("job-1", "s1")] + assert body["call"]["transcript_artifact"] is None + assert body["call"]["recording_artifacts"] == [] + + log_events = [r for r in transport.event_records if r.get("type") == "log"] + assert any( + r["payload"]["level"] == "error" and "un-acked transcript" in r["payload"]["message"] + for r in log_events + ) + assert any( + r["payload"]["level"] == "error" and "un-acked recording" in r["payload"]["message"] + for r in log_events + ) + + asyncio.run(scenario()) + + +def test_pre_run_provision_failure_emits_the_terminal_before_closing_the_pool() -> None: + # `test_finish_emits_the_terminal_before_...` + # only drives the COMPLETED path -- a pre-run failure (`_fail` -> `_finish`) is a structurally + # different entry into `_finish` and needs its own ordering check. + async def scenario() -> None: + order: list[str] = [] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + return response + + class FailingProvisioner(FakeProvisioner): + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + del bundle, source, bundle_dir, work_directory, contract, instances + raise ProcessRuntimeError("build", "build_failed", "synthetic", process="agent") + + async def close(self, *, work_directory: Path) -> None: + await super().close(work_directory=work_directory) + order.append("pool_closed") + + harness = _build_harness(scenarios=[], instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + harness.deps.build_provider = lambda: FailingProvisioner(instances=1) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert order[:2] == ["terminal_delivered", "pool_closed"] + terminals = transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["failure"]["domain"] == "agent" + + asyncio.run(scenario()) + + +# ================================================================================================= +# Additional tests -- the wire retargeted the fence-on-final-drain test to the pre-drain +# check; flush_terminal's and drain()'s bounded loops now cover for each other; the +# post-terminal wire block had no deadline. +# ================================================================================================= + + +def test_fence_on_the_manifest_push_still_exits_fenced_with_the_terminal_already_delivered() -> None: + # `flush_terminal` now delivers the terminal-carrying POST ahead of `drain()`, so a fence + # on that POST is caught by the PRE-drain `is_fenced` check in `_finish`, not the post-drain + # `if fenced: return EXIT_FENCED` line -- `test_fence_landing_on_the_final_ + # drain_still_exits_fenced` no longer exercises that check for the reason its own comment + # claims. A fence that only starts on `/manifest/` lands strictly inside `drain()` itself + # (drain()'s own `push_manifest` call), after a clean `flush_terminal`, so `drain()`'s return + # value -- and therefore only the post-drain check -- is what decides the exit code here. Also + # pins the observation that a fenced attempt can still have already delivered its terminal, + # deliberately rather than by accident. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, instances=1, fence_on_url_substring="/manifest/", + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_FENCED + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # delivered before the fence ever landed + assert terminals[0]["payload"]["stage"] == "completed" + assert harness.provisioner.closed is True + + asyncio.run(scenario()) + + +def test_drain_alone_delivers_a_pre_run_backlog_larger_than_one_batch() -> None: + # On a pre-run terminal (`_fail` -> `_finish` with no `scheduler_result`), the wire's + # `flush_terminal` is never called at all -- `drain()`'s own bounded loop is the ONLY delivery + # mechanism for whatever accumulated beforehand. `test_drain_loops_past_a_backlog_...` only + # drives a post-run COMPLETED path, where `flush_terminal` already drains everything first and + # masks a collapsed `drain()` loop. A `ScenarioSource` that logs a large pre-run backlog before + # raising a typed pre-allocation failure reproduces the shape end to end, on the one path where + # collapsing `drain()`'s loop to a single flush cannot be covered for by anything else. + async def scenario() -> None: + class ChattyPreRunScenarioSource: + def __init__(self, *, chatter_count: int) -> None: + self._chatter_count = chatter_count + + async def build( + self, job: HarnessJob, bundle: Any, scenarios_client: he.ScenariosClient, *, + pool: Any, world_factory: Any, bundle_dir: Path, + ) -> list[FakeScenario]: + del job, bundle, scenarios_client, world_factory, bundle_dir + adapter = pool._outbound # no adapter seam on ScenarioSource itself + for i in range(self._chatter_count): + await adapter.log(level="info", message=f"pre-run chatter {i}") + raise he.ScenarioPreallocationError(None) + + harness = _build_harness(scenarios=[], instances=1) + harness.deps.scenario_source = ChattyPreRunScenarioSource(chatter_count=260) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # must not be stranded behind the pre-run backlog + assert terminals[0]["payload"]["stage"] == "failed" + + chatter_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and "pre-run chatter" in record["payload"].get("message", "") + ] + assert len(chatter_logs) == 260 # the whole backlog drained, not just the first batch + + asyncio.run(scenario()) + + +def test_flush_terminal_alone_must_deliver_the_terminal_before_a_skipped_receipt_under_backlog() -> None: + # The converse of the previous test -- on a cancel-mid-run path, `flush_terminal` is the + # ONLY thing standing between "terminal not yet on the wire" and `emit_skipped_receipts` + # pushing a receipt straight to `/results/` (that push is not gated on event delivery at all). + # A backlog bigger than one batch, queued before the cancel is even noticed, means a collapsed + # `flush_terminal` loop delivers only the first batch and leaves the terminal pending -- the + # skipped receipt for "second" then reaches the platform BEFORE the terminal does, even though + # `drain()`'s own (intact) loop mops up the rest a moment later. Ordering, not delivery count, + # is what only `flush_terminal`'s loop can guarantee here. + async def scenario() -> None: + order: list[str] = [] + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + + class OrderTrackingTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + response = super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + if ( + "/events/" in url and method == "POST" + and isinstance(data, (bytes, bytearray)) + and b'"type":"terminal"' in bytes(data) + ): + order.append("terminal_delivered") + if ( + "/results/" in url and method == "POST" + and json_body is not None and json_body.get("status") == "skipped" + ): + order.append("skipped_receipt_delivered") + return response + + class ChattyCancelingCallRunner: + def __init__( + self, adapter: he.OutboundAdapter, *, cancel_path: Path, cancel_on_scenario: str, + chatter_count: int, + ) -> None: + self._adapter = adapter + self._cancel_path = cancel_path + self._cancel_on_scenario = cancel_on_scenario + self._chatter_count = chatter_count + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + if scenario.scenario_key == self._cancel_on_scenario: + for i in range(self._chatter_count): + await self._adapter.log(level="info", message=f"chatter {i}") + self._cancel_path.write_text( + json.dumps({"reason": "user_canceled"}), encoding="utf-8" + ) + now = _rfc3339(datetime.now(timezone.utc)) + return CallOutcome( + calls=( + Call( + name="tool", arguments={}, result="ok", ok=True, error="", + refused=False, at=0.0, + ), + ), + turns=1, started_at=now, ended_at=now, duration_ms=10, + transcript_artifact=None, recording_artifacts=(), + ) + + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + transport = OrderTrackingTransport() + harness.deps.build_transport = lambda: transport + harness.deps.build_call_runner = lambda adapter, context: ChattyCancelingCallRunner( + adapter, cancel_path=harness.deps.cancel_path, cancel_on_scenario="first", + chatter_count=260, + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + statuses = {key[1]: body["status"] for key, body in transport.receipts.items()} + assert statuses.get("first") == "passed" + assert statuses.get("second") == "skipped" + + assert order == ["terminal_delivered", "skipped_receipt_delivered"] + + asyncio.run(scenario()) + + +def test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window() -> None: + # `emit_skipped_receipts`/`receipt()` push with `deadline=None` -- a degraded-but-alive + # events channel could previously retry every skipped scenario's receipt for the full + # `RetryPolicy` budget regardless of how much of the flush window was already spent, well past + # the point the gateway tears the sandbox down. A tiny window plus a slow + # `emit_skipped_receipts` reproduces the shape without a real multi-second stall dominating the + # suite: `asyncio.wait_for`'s own cancellation cuts the sleep short well before it would run out. + async def scenario() -> None: + scenarios = [ + FakeScenario("first", "platform-first", [FakeSubGoal("holds", True)]), + FakeScenario("second", "platform-second", [FakeSubGoal("holds", True)]), + ] + harness = _build_harness(scenarios=scenarios, cancel_on_scenario="first", instances=1) + harness.deps.flush_window_seconds = 0.2 + + original_emit = HostedScheduler.emit_skipped_receipts + + async def slow_emit(self: HostedScheduler, result: RunResult) -> None: + await asyncio.sleep(2.0) + await original_emit(self, result) + + HostedScheduler.emit_skipped_receipts = slow_emit # type: ignore[method-assign] + started = time.monotonic() + try: + code = await he.run_job( + harness.job_path, harness.source, harness.output, deps=harness.deps + ) + finally: + HostedScheduler.emit_skipped_receipts = original_emit + elapsed = time.monotonic() - started + + assert elapsed < 1.5, f"post-terminal wire block was not bounded: {elapsed:.2f}s" + assert code == he.EXIT_OK # the terminal was already delivered before the window ran out + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "canceled" + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses.get("first") == "passed" + assert "second" not in statuses # window ran out before the skipped receipt could go + + assert harness.provisioner.closed is True # the finally backstop still closes the pool + + asyncio.run(scenario()) + + +# ================================================================================================= +# p12: scenario_source.py wiring -- item 4. Writes real scenario documents (`folder.py`'s own +# on-disk layout) straight into `harness.bundle_dir`, matching `scenario_source.py`'s +# `SCENARIOS_DIRNAME` constant. Deliberately its own tiny writer rather than importing +# `test_scenario_source.py`'s `_write_scenario` (no cross-test-module private-helper imports). +# +# §2e preflight (`bundle_file_unlisted`) rejects any bundle file absent from the manifest's +# `files[]` -- exactly the CONTRACT QUESTIONS obligation the report calls out for a real Scenario +# Generation Contract bundle author. So a scenario-bearing bundle for these tests cannot just drop +# files under `bundle_dir/scenarios/` after `_write_bundle` has already sealed the manifest; the +# scenario files have to be hashed into `files[]` and the bundle re-sealed, the same way +# `test_process_preflight.py`'s own `_build_bundle(extra_files=...)` does it. +# ================================================================================================= + + +def _scenario_doc_files( + name: str, *, scenario_key: str, scenario_id: str = "", sub_goals: list[str] | None = None, + setup_code: str = "", checks: dict[str, str] | None = None, +) -> dict[str, bytes]: + """One scenario folder's contents as {relative path: bytes} -- fed to + `_write_bundle_with_scenario_files` below rather than written straight to disk, so every byte + can be hashed into the manifest's `files[]` before the bundle is sealed.""" + body = { + "name": name, "scenario_key": scenario_key, "scenario_id": scenario_id, + "sub_goals": sub_goals or [], + } + prefix = f"{SCENARIOS_DIRNAME}/{name}" + files = {f"{prefix}/scenario.json": json.dumps(body).encode("utf-8")} + if setup_code: + files[f"{prefix}/setup.py"] = setup_code.encode("utf-8") + for goal_name, code in (checks or {}).items(): + files[f"{prefix}/checks/{goal_name}.py"] = code.encode("utf-8") + return files + + +def _write_bundle_with_scenario_files(root: Path, scenario_files: dict[str, bytes]) -> EnvironmentBundleV2: + """`_write_bundle`, plus `scenario_files` hashed into `files[]` and the digest re-sealed over + all of it -- otherwise every one of these trips `bundle_file_unlisted` at preflight, before + `scenario_source.build()` is ever reached.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL, **scenario_files} + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", "migrations": ["db/schema.sql"], "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +def test_bundle_without_scenarios_keeps_the_notwired_regression() -> None: + # item 4: a bundle that does not carry a `scenarios/` directory must keep behaving exactly as + # it did before this adapter existed -- the default `NotWiredScenarioSource`'s typed failure, + # never a crash and never a silently-empty run. + async def scenario() -> None: + harness = _build_harness(scenarios=[], instances=1, use_default_scenario_source=True) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "platform_sync" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() -> None: + # item 4 + 5c (end to end): the presence test flips the default over to the real + # `BundleScenarioSource` -- no `FakeScenarioSource` involved anywhere in this test. One + # deterministic sub_goal that genuinely holds against the fake world, so this proves a real + # COMPLETED pass, not just that the vacuous-pass guard fired. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + # `scenario_id` non-empty here on purpose -- see + # `test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema` just below for + # the (newly discovered, load-bearing) reason an EMPTY one cannot be used to + # prove a receipt actually arrives. + "passing", scenario_key="passing", scenario_id="platform-passing", + sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["failure"] is None + + # terminal last: the terminal record is the final event this run ever pushed. + assert harness.transport.event_records[-1].get("type") == "terminal" + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed"} # the real scheduler actually ran it, and it held + + asyncio.run(scenario()) + + +def test_empty_scenario_key_from_bundle_document_fails_cleanly_via_existing_validation() -> None: + # Mutation table item 2: "empty-key fixture -> typed failure", at the FULL integration level -- + # a hand-written document with an empty `scenario_key` flows verbatim through this adapter + # (work item 3: never synthesized) into the scheduler's OWN pre-existing defense + # (`_validate_scenario_entry`, untouched by this task), which must catch it as a typed FAILED + # terminal rather than an `AttributeError` deep in the scheduler. `test_scenario_source.py` + # covers the reader's verbatim carry and the mutation that would break it; this is the + # end-to-end proof that the two halves agree. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, _scenario_doc_files("s1", scenario_key="", sub_goals=[]) + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "environment" + assert failure["code"] == "scenario_preallocation_failed" + assert "scenario_key" in failure["message"] + + asyncio.run(scenario()) + + +def test_mutation_adapter_off_makes_the_e2e_test_fail() -> None: + # Mutation table item 1: "adapter-off -> e2e test fails". Patches `bundle_has_scenarios` (as + # seen from `hosted_entrypoint.py`'s own namespace, where it was imported) to always report + # "no scenarios here" -- simulating hard-rule edit (b) never having been made, i.e. the wiring + # `if isinstance(...) and bundle_has_scenarios(...)` guard permanently failing closed. + # + # R1-6 fold-in (p12-review-r1.md LOW finding): the original version of this test asserted the + # MUTANT's own failure terminal directly -- true, but it never actually ran the `stage == + # "completed"` assertion that is the real kill. This version runs the REAL + # `test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present` under the + # patch and checks THAT it fails. `mock.patch.object` restores the original function in its own + # `finally` regardless of how the inner call ends -- no manual restore bookkeeping needed. + with mock.patch.object(he, "bundle_has_scenarios", lambda bundle_dir: False): + try: + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() + except AssertionError: + pass + else: + raise AssertionError( + "adapter-off mutant did not fail " + "test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present " + "(no pytest.raises here -- this file runs stand-alone via TESTS, per its own " + "module docstring)" + ) + + # Restored: the real test passes again. + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present() + + +def test_bundle_scenario_id_is_assigned_by_registration_and_receipt_now_delivers() -> None: + # p13 UPDATE of the former `test_empty_scenario_id_receipt_is_dropped_by_the_wire_schema` + # (p12): that test pinned a real gap -- `outbound.py`'s `ResultReceiptDraft` schema requires + # `scenario_id` non-empty (pydantic `min_length=1`), and before this task nothing ever filled + # it in, so a bundle-sourced scenario's receipt was silently dropped. Registration + # (`register_with_platform`, scenario_source.py) now runs between load and the scheduler and + # OVERWRITES `scenario_id` with the platform-assigned one before `BundleScenarioSource.build` + # ever returns -- the document is written with `scenario_id=""` here specifically to prove the + # id on the wire came from the (fake) platform's provision response, not the document. See + # `test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely` just below for + # the property this test used to pin, preserved on the path that never registers at all. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "passing", scenario_key="passing", scenario_id="", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" + assert payload["scenario_counts"]["passed"] == 1 + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"passing": "passed"} # the receipt DELIVERS now -- no drop. + + (_, body), = [ + (key, body) for key, body in harness.transport.receipts.items() if key[1] == "passing" + ] + # `FakeTransport`'s fake platform assigns `f"platform-{scenario_key}"` -- confirms the id + # on the wire is the PLATFORM's, not the document's own (empty) one. + assert body["scenario_id"] == "platform-passing" + + error_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and record["payload"].get("level") == "error" + and "ResultReceiptDraft" in record["payload"].get("message", "") + ] + assert error_logs == [] # no drop, so no drop log either. + + asyncio.run(scenario()) + + +def test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely() -> None: + # Preserves the property the pre-p13 pinning test proved: a scenario that reaches the + # scheduler with an empty `scenario_id` (never pre-allocated) still has its receipt rejected by + # `ResultReceiptDraft`'s own schema (`min_length=1`) and DROPPED, loudly, rather than crashing + # the job or silently delivering a receipt the platform would 422 anyway. Through + # `BundleScenarioSource` this is now unreachable (registration always assigns a real id or the + # job fails first) -- so this is exercised through `FakeScenarioSource`'s injected-scenario + # path instead, which never calls `register_with_platform` at all (an "unregistered" scenario + # source, same shape as a future ScenarioSource that also skips pre-allocation). + async def scenario() -> None: + scenarios = [FakeScenario("passing", "", [FakeSubGoal("holds", True)])] + harness = _build_harness(scenarios=scenarios, instances=1) # FakeScenarioSource, as usual + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + payload = terminals[0]["payload"] + assert payload["stage"] == "completed" # the job itself does not fail + assert payload["scenario_counts"]["passed"] == 1 # ...and reports the scenario as passed... + + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {} # ...yet no receipt for it ever reached the platform. + + error_logs = [ + record for record in harness.transport.event_records + if record.get("type") == "log" and record["payload"].get("level") == "error" + and "ResultReceiptDraft" in record["payload"].get("message", "") + ] + assert len(error_logs) == 1 # the drop is at least loud, not silent -- but still a drop. + + asyncio.run(scenario()) + + +def test_registration_response_mismatch_reaches_the_typed_platform_sync_terminal() -> None: + # p13: a provision response that fails `_scenario_ids_by_key`'s guards (scenario_source.py -- + # here, naming NO scenario_key at all, so every submitted one is "missing") must fail the + # whole job through the SAME typed `validating_scenarios`/`platform_sync` terminal + # `run_job`'s existing `except (ScenarioSourceNotWired, ScenarioPreallocationError)` clause + # already produces for every other pre-allocation failure -- no new except clause needed. The + # scheduler must never run: no receipt for the scenario ever reaches the platform. + async def scenario() -> None: + class MismatchedProvisionTransport(FakeTransport): + def request( + self, method: str, url: str, *, headers: dict[str, str], + json_body: dict[str, Any] | None = None, data: bytes | Any | None = None, + timeout: float = 30.0, + ) -> ob.TransportResponse: + if ( + method == "POST" and "/scenarios/" in url and json_body is not None + and json_body.get("operation") == "provision" + ): + self.calls.append({"method": method, "url": url, "headers": dict(headers)}) + self.scenarios_calls.append((url, json_body)) + return ob.TransportResponse( + 200, {"result": {"run_test_id": "run-test-1", "scenarios": []}}, {}, + ) + return super().request( + method, url, headers=headers, json_body=json_body, data=data, timeout=timeout, + ) + + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "passing", scenario_key="passing", scenario_id="", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ), + ), + ) + transport = MismatchedProvisionTransport() + harness.deps.build_transport = lambda: transport + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["stage"] == "validating_scenarios" + assert failure["domain"] == "platform_sync" + assert failure["code"] == "scenario_preallocation_failed" + + assert transport.receipts == {} # the scheduler never ran -- registration failed first + # `begin` must never have been attempted -- the provision-side guard stops it first. + assert not any(body.get("operation") == "begin" for _, body in transport.scenarios_calls) + + asyncio.run(scenario()) + + +def test_injected_scenario_source_always_wins_over_the_bundle_adapter() -> None: + # item 4: even when the bundle ALSO carries a valid `scenarios/` directory, an explicitly + # injected `ScenarioSource` must be used untouched -- the presence test only ever applies to + # the untouched default. + async def scenario() -> None: + injected = [FakeScenario("from-fake", "platform-from-fake", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=injected, instances=1, # FakeScenarioSource, as usual -- not the default + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, _scenario_doc_files("from-bundle", scenario_key="from-bundle", sub_goals=[]) + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + statuses = {key[1]: body["status"] for key, body in harness.transport.receipts.items()} + assert statuses == {"from-fake": "passed"} # the bundle's own scenario never ran + + asyncio.run(scenario()) + + +# ================================================================================================= +# R1-1 (CRITICAL, p12-review-r1.md) -- through the REAL `run_job`, on a sealed, preflight-clean +# bundle: a scenario document must never be able to set the guest's own exit code, and an unreadable +# or malformed scenario file must never escape as an unhandled exception with zero terminal events. +# The chmod-000 "unreadable file" trigger is proven directly against `scenario_source.py` in +# `test_scenario_source.py` instead of here -- see that file's R1-1 section docstring for why the +# full bundle/preflight pipeline cannot exercise it without touching `process_preflight.py`. +# ================================================================================================= + + +def test_storeless_bundle_run_with_state_free_checks_passes_end_to_end() -> None: + # The relaxed store rule end-to-end: a bundle with zero postgres capabilities clears + # preflight, the real `ProcessWorldFactory` hands the scheduler a `StorelessWorld` (the fake + # provisioner's runtimes carry no endpoints and build.json records no stores), and a scenario + # whose checks never touch state passes exactly as it would against a postgres world. + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [FakeSubGoal("holds", True)])] + harness = _build_harness( + scenarios=scenarios, bundle_writer=_write_storeless_bundle, + build_output={"stores": []}, + ) + harness.deps.build_world_factory = he.ProcessWorldFactory + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + receipt = harness.transport.receipts[("job-1", "s1")] + assert receipt["status"] == "passed" + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + +def test_storeless_bundle_check_touching_state_fails_typed_never_crashes() -> None: + # A check calling `world.state()` against a storeless world must land in the scheduler's + # typed `world_usage` lane — an errored receipt naming the missing store — while the run + # itself still completes cleanly. + @dataclass + class StateTouchingSubGoal: + name: str + judged: str = "yes" + + def check(self, world: Any, calls: Any) -> object: + del calls + world.state() + return None + + async def scenario() -> None: + scenarios = [FakeScenario("s1", "platform-s1", [StateTouchingSubGoal("reads-state")])] + harness = _build_harness( + scenarios=scenarios, bundle_writer=_write_storeless_bundle, + build_output={"stores": []}, + ) + harness.deps.build_world_factory = he.ProcessWorldFactory + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + receipt = harness.transport.receipts[("job-1", "s1")] + assert receipt["status"] == "errored" + assert receipt["failure"]["code"] == "world_usage" + assert "no SQL store" in receipt["failure"]["message"] + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + assert terminals[0]["payload"]["stage"] == "completed" + + asyncio.run(scenario()) + + +def test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure() -> None: + # Before the fix: `sys.exit(0)` at module level inside `setup.py` propagated as a raw + # `SystemExit` straight out of `run_job` -- the guest process itself would exit 0 with ZERO + # terminal events (a "clean terminal that never happened", per spine §0.6), on a bundle whose + # every byte was hashed into the manifest and §2e preflight passed. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "s1", scenario_key="s1", sub_goals=[], setup_code="import sys\nsys.exit(0)\n", + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + # The GUEST's own exit code -- EXIT_OK means "a terminal was reached and flushed", the + # same meaning it carries for any other typed FAILED terminal, never the scenario's own + # sys.exit(0) leaking through `run_job`'s return value. + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 # a terminal event WAS delivered -- not an empty event stream + payload = terminals[0]["payload"] + assert payload["stage"] == "failed" + failure = payload["failure"] + assert failure["domain"] == "environment" + assert failure["stage"] == "validating_scenarios" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_module_level_sys_exit_three_in_setup_does_not_hijack_the_guests_exit_code() -> None: + # EXIT_FENCED == 3: before the fix, `sys.exit(3)` here was indistinguishable from the guest + # itself choosing to exit fenced -- the platform would read an ordinary scenario content defect + # as a fenced/superseded attempt instead. + async def scenario() -> None: + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files( + bundle_dir, + _scenario_doc_files( + "s1", scenario_key="s1", sub_goals=[], setup_code="import sys\nsys.exit(3)\n", + ), + ), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + assert code != he.EXIT_FENCED # explicit: the scenario's own exit code did not leak through + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == "environment" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +def test_non_utf8_setup_file_is_contained_as_a_typed_failure_not_an_escape() -> None: + # Before the fix: a raw `UnicodeDecodeError` from `_read_text`'s `.read_text(encoding="utf-8")` + # propagated straight out of `run_job` -- per spine §0.6 a non-zero exit with no terminal event + # reads as `infrastructure` and gets retried to exhaustion, even though this is a deterministic + # content defect that will never succeed on retry. The non-UTF-8 bytes hash and seal fine (§2e + # preflight only hashes raw bytes, never decodes) -- only this module's own `.read_text()` call + # ever attempts to decode them. + async def scenario() -> None: + files = _scenario_doc_files("s1", scenario_key="s1", sub_goals=[]) + files[f"{SCENARIOS_DIRNAME}/s1/setup.py"] = b"def setup(world):\n return '\xff\xfe'\n" + harness = _build_harness( + scenarios=[], instances=1, use_default_scenario_source=True, + bundle_writer=lambda bundle_dir: _write_bundle_with_scenario_files(bundle_dir, files), + ) + code = await he.run_job(harness.job_path, harness.source, harness.output, deps=harness.deps) + assert code == he.EXIT_OK + + terminals = harness.transport.terminal_events() + assert len(terminals) == 1 + failure = terminals[0]["payload"]["failure"] + assert failure["domain"] == "environment" + assert failure["stage"] == "validating_scenarios" + assert failure["code"] == "scenario_preallocation_failed" + + asyncio.run(scenario()) + + +TESTS = [ + test_resolve_parallelism_reads_the_raw_value_without_clamping, + test_out_of_range_parallelism_is_rejected_by_preflight_not_clamped, + test_job_secret_purposes_maps_alias_to_purpose, + test_peek_secret_values_reads_without_deleting, + test_peek_secret_values_missing_file_is_empty, + test_peek_target_provider_secret_values_filters_by_purpose_and_keeps_the_alias, + test_peek_target_provider_secret_values_missing_file_is_empty, + test_peek_target_provider_secret_values_drops_an_alias_with_no_purpose_entry, + test_default_build_call_runner_returns_notwired_for_a_non_livekit_connector, + test_default_build_call_runner_returns_notwired_for_retell_and_auto_too, + test_default_build_call_runner_returns_a_real_call_runner_impl_for_livekit, + test_call_runner_context_is_threaded_with_real_job_bundle_secrets_and_evidence_seam, + test_row_counts_for_capability_returns_the_matching_store, + test_row_counts_for_capability_raises_when_the_capability_is_absent, + test_cancel_state_reads_reason_from_file, + test_world_pool_serializes_concurrent_provider_calls_end_to_end, + test_scenarios_client_provision_unwraps_the_result_envelope, + test_scenarios_client_provision_and_begin_hit_the_same_single_url, + test_scenarios_client_fencing_latches_the_shared_channel_state, + test_capabilities_failure_exits_boot_failure_with_no_channel_and_no_event, + test_preflight_rejection_reaches_a_failed_terminal_event_before_any_provision, + test_hosted_fenced_error_stops_emitting_and_exits_3_with_no_terminal_event, + test_cancel_mid_run_synthesizes_a_skipped_receipt_for_the_unstarted_scenario, + test_fenced_run_result_emits_zero_skipped_receipts, + test_emit_skipped_receipts_failure_does_not_lose_the_terminal_or_the_exit_code, + test_cancel_mid_run_with_ttl_exceeded_reports_that_reason, + test_malformed_job_json_exits_crashed_with_no_terminal_event, + test_world_pool_exhaustion_reaches_a_failed_terminal_world_pool_exhausted, + test_fence_landing_on_the_final_drain_still_exits_fenced, + test_fence_from_scenarios_client_exits_fenced_not_crashed, + test_scenarios_channel_uses_bearer_auth_never_api_key, + test_process_world_factory_raises_when_stores_recorded_but_no_endpoint, + test_process_world_factory_builds_storeless_world_when_no_endpoint_and_no_stores, + test_process_world_factory_raises_when_build_json_has_no_matching_store, + test_build_json_two_stores_emit_two_baseline_frozen_events, + test_build_json_degrade_payload_matches_the_recorded_values, + test_build_json_fixed_port_at_w1_does_not_crash, + test_e2e_two_scenarios_one_pass_one_fail_reaches_completed_and_exits_0, + test_pool_close_backstop_runs_even_when_scenario_source_raises_untyped, + test_process_runtime_error_uses_the_carried_domain_over_the_fallback_map, + test_finish_emits_the_terminal_before_closing_the_pool, + test_redaction_end_to_end_secret_never_crosses_any_channel, + test_drain_loops_past_a_backlog_larger_than_one_batch_and_still_delivers_the_terminal, + test_call_aborted_with_no_ended_at_still_produces_a_receipt, + test_metadata_only_artifact_level_refuses_transcript_upload_end_to_end, + test_traces_artifact_level_refuses_recording_upload_end_to_end, + test_secrets_unlink_failure_after_the_terminal_does_not_lose_the_terminal_event, + test_events_channel_dying_on_the_final_drain_exits_terminal_undelivered, + test_platform_rejecting_the_terminal_event_exits_terminal_undelivered, + test_receipt_rejection_by_the_platform_is_logged, + test_receipt_failure_message_is_capped_like_the_terminal_message, + test_cancel_before_provision_skips_provisioning_entirely, + test_evaluation_wire_coerces_an_int_score_to_float, + test_receipt_nulls_an_unacked_transcript_artifact_and_logs_it, + test_pre_run_provision_failure_emits_the_terminal_before_closing_the_pool, + test_fence_on_the_manifest_push_still_exits_fenced_with_the_terminal_already_delivered, + test_drain_alone_delivers_a_pre_run_backlog_larger_than_one_batch, + test_flush_terminal_alone_must_deliver_the_terminal_before_a_skipped_receipt_under_backlog, + test_post_terminal_wire_block_is_bounded_by_the_remaining_flush_window, + test_bundle_without_scenarios_keeps_the_notwired_regression, + test_default_scenario_source_wires_the_bundle_adapter_when_scenarios_present, + test_empty_scenario_key_from_bundle_document_fails_cleanly_via_existing_validation, + test_mutation_adapter_off_makes_the_e2e_test_fail, + test_bundle_scenario_id_is_assigned_by_registration_and_receipt_now_delivers, + test_unregistered_scenario_with_empty_scenario_id_receipt_still_drops_safely, + test_registration_response_mismatch_reaches_the_typed_platform_sync_terminal, + test_injected_scenario_source_always_wins_over_the_bundle_adapter, + test_module_level_sys_exit_zero_in_setup_is_contained_as_a_typed_failure, + test_module_level_sys_exit_three_in_setup_does_not_hijack_the_guests_exit_code, + test_non_utf8_setup_file_is_contained_as_a_typed_failure_not_an_escape, +] + + +if __name__ == "__main__": + failures = 0 + for test_fn in TESTS: + started = time.monotonic() + try: + test_fn() + except Exception as exc: # noqa: BLE001 - a direct-invocation runner, not pytest + failures += 1 + print(f"FAIL {test_fn.__name__}: {type(exc).__name__}: {exc}") + else: + print(f"ok {test_fn.__name__} ({time.monotonic() - started:.2f}s)") + print(f"\n{len(TESTS) - failures}/{len(TESTS)} passed") + raise SystemExit(1 if failures else 0) diff --git a/tests/harness/test_hosted_scheduler.py b/tests/harness/test_hosted_scheduler.py new file mode 100644 index 00000000..5208a9a3 --- /dev/null +++ b/tests/harness/test_hosted_scheduler.py @@ -0,0 +1,2486 @@ +"""`hosted_scheduler.py` against in-memory fakes — no real provisioner, no real postgres. + +`asyncio.run` drives every `async def` seam here, matching `test_process_runtime.py`'s own +convention (no pytest-asyncio dependency in this repo). +""" + +from __future__ import annotations + +import asyncio +import contextlib +import random +import threading +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from fi.alk.harness import hosted_scheduler as hs +from fi.alk.harness.outbound import ( + ChannelError, + ChannelOutcome, + HostedAttemptSupersededError, + HostedChannelFailedError, + HostedFencedError, +) +from fi.alk.harness.process_runtime import EnvironmentRuntime, ProcessRuntimeError, RuntimeState +from fi.alk.harness.world.errors import ( + WorldReadOnly, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) + +# --- fakes --------------------------------------------------------------------------------- + + +def _runtime(index: int, state: RuntimeState = RuntimeState.READY) -> EnvironmentRuntime: + return EnvironmentRuntime( + runtime_id=f"digest:w{index}", world_index=index, bundle_digest="digest", state=state + ) + + +class FakeProvisioner: + """Mirrors `ProcessRuntimeProvider`'s real async shape, including the two properties the + review named as fidelity gaps: `reset_scripts` lets a test script one world's next N reset + outcomes (anything unscripted resets clean to READY), and every provider call — including + `healthy()` (R13: spine v1.12 §4.5b folded it into the same non-reentrant set) — goes through + `_serialized`, which yields (`await asyncio.sleep(0)` — so a genuine overlap has a real + chance to interleave) and records any overlapping call into `overlaps`, matching "not + reentrant" (B1/B2's own regression test). + + `_serialized` used to `assert not self._busy` in-band — but that assertion fires INSIDE + `_reconcile`'s own `except Exception` (a reconcile must never crash the pool) or `lease()`'s + `except Exception as exc: reset_exc = exc`, so production swallows it and the calling test + never sees a failure. Recording into `overlaps` and asserting on it from the test's OWN frame + is what actually makes a `_provider_lock` regression observable.""" + + def __init__( + self, instances: int, *, reset_scripts: dict[int, list[RuntimeState]] | None = None + ) -> None: + self.instances = instances + self.reset_scripts = reset_scripts or {} + self.provision_calls = 0 + self.reset_calls = 0 + self.healthy_calls = 0 + self.closed = False + self._runtimes = {i: _runtime(i) for i in range(instances)} + self.overlaps: list[str] = [] + self._in_flight: list[str] = [] + + @contextlib.asynccontextmanager + async def _serialized(self, label: str): + if self._in_flight: + self.overlaps.append(f"{self._in_flight[-1]} overlapped {label}") + self._in_flight.append(label) + try: + await asyncio.sleep(0) + yield + finally: + # `remove` (not `pop`) — cancellation (M6) can unwind these out of call order. + self._in_flight.remove(label) + + async def provision( + self, bundle: Any, *, source: Path, bundle_dir: Path, work_directory: Path, + contract: Any | None = None, instances: int = 1, + ) -> list[EnvironmentRuntime]: + async with self._serialized("provision"): + self.provision_calls += 1 + for index in range(instances): + if index not in self._runtimes or self._runtimes[index].state in ( + RuntimeState.STOPPED, RuntimeState.UNHEALTHY, + ): + self._runtimes[index] = _runtime(index, RuntimeState.READY) + return [self._runtimes[index] for index in range(instances) if index in self._runtimes] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + script = self.reset_scripts.get(runtime.world_index) + runtime.state = script.pop(0) if script else RuntimeState.READY + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory: Path) -> None: + async with self._serialized("close"): + self.closed = True + + +class InMemoryWorld: + """The six-verb surface, backed by a plain dict instead of postgres — enough to exercise the + scheduler's own control flow without a `PostgresStore`.""" + + def __init__(self, world_index: int, rng: random.Random, *, read_only: bool = False) -> None: + self.world_index = world_index + self.rng = rng + self._read_only = read_only + self.rows: dict[str, list[dict[str, Any]]] = {} + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return dict(self.rows) if table is None else {table: list(self.rows.get(table, []))} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + if self._read_only: + raise WorldReadOnly("read-only world") + self.rows.setdefault(collection, []).append(record) + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + if self._read_only: + raise WorldReadOnly("read-only world") + return 0 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + if self._read_only: + raise WorldReadOnly("read-only world") + return 0 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> hs.Call: + if self._read_only: + raise WorldReadOnly("read-only world") + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + return [] + + def read_only(self) -> "InMemoryWorld": + return InMemoryWorld(self.world_index, self.rng, read_only=True) + + +class FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> InMemoryWorld: + return InMemoryWorld(runtime.world_index, rng) + + +@dataclass +class FakeSubGoal: + name: str + fn: Any + judged: str = "" + + def check(self, world: Any, calls: Any) -> object: + return self.fn(world, calls) + + +@dataclass +class FakeScenario: + scenario_key: str + scenario_id: str + sub_goals: list[FakeSubGoal] = field(default_factory=list) + setup_fn: Any = lambda world: None + ready_fn: Any = lambda world: None + + def setup(self, world: Any) -> object: + return self.setup_fn(world) + + def ready(self, world: Any) -> object: + return self.ready_fn(world) + + +class FakeCallRunner: + def __init__(self, outcomes: dict[str, Any]) -> None: + self.outcomes = outcomes + self.calls: list[tuple[str, int]] = [] + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + self.calls.append((scenario.scenario_key, runtime.world_index)) + outcome = self.outcomes[scenario.scenario_key] + if isinstance(outcome, Exception): + raise outcome + return outcome + + +class FakeOutbound: + def __init__(self) -> None: + self.events: list[tuple[str, dict[str, Any]]] = [] + self.receipts: list[hs.ResultReceipt] = [] + + async def scenario_started(self, *, scenario_key: str, world_index: int, scenario_attempt: int) -> None: + self.events.append(("scenario_started", {"scenario_key": scenario_key, "world_index": world_index, "scenario_attempt": scenario_attempt})) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + self.events.append(("scenario_retried", {"scenario_key": scenario_key, "from_world": from_world, "to_world": to_world})) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + self.events.append(("world_unhealthy", {"world_index": world_index, "cause": cause})) + + async def log(self, *, level: str, message: str) -> None: + self.events.append(("log", {"level": level, "message": message})) + + async def receipt(self, receipt: hs.ResultReceipt) -> None: + self.receipts.append(receipt) + + +class FailingOutbound(FakeOutbound): + """B3 regression fixture: a subset of methods raise, standing in for a dead transport.""" + + def __init__(self, *, fail_on: set[str]) -> None: + super().__init__() + self._fail_on = fail_on + + async def scenario_started(self, **kwargs: Any) -> None: + if "scenario_started" in self._fail_on: + raise ConnectionError("outbound transport down") + await super().scenario_started(**kwargs) + + async def receipt(self, receipt: hs.ResultReceipt) -> None: + if "receipt" in self._fail_on: + raise ConnectionError("outbound transport down") + await super().receipt(receipt) + + +def _call_outcome(turns: int = 1, calls: tuple[hs.Call, ...] = ()) -> hs.CallOutcome: + return hs.CallOutcome( + calls=calls, turns=turns, started_at="2026-08-25T00:00:00.000Z", + ended_at="2026-08-25T00:00:05.000Z", duration_ms=5000, + ) + + +def _pool( + instances: int, *, provisioner: Any | None = None, + reset_scripts: dict[int, list[RuntimeState]] | None = None, + outbound: Any | None = None, +) -> tuple[hs.WorldPool, FakeProvisioner]: + fake = provisioner or FakeProvisioner(instances, reset_scripts=reset_scripts) + pool = hs.WorldPool( + fake, bundle=object(), source=Path("/work/source"), bundle_dir=Path("/work/bundle"), + work_directory=Path("/work"), instances=instances, outbound=outbound, + ) + return pool, fake + + +# --- WorldPool ------------------------------------------------------------------------------- + + +def test_lease_skips_reset_for_a_freshly_provisioned_world_but_not_the_next_lease() -> None: + # m9: a world just handed back by `provision()` is already at the sealed baseline — the + # first lease must not pay for a redundant reset, but a world that has already been used + # once resets normally on its next lease. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + world_index, runtime = await pool.lease() + assert world_index == 0 + assert runtime.state is RuntimeState.READY + assert provisioner.reset_calls == 0 + await pool.release(world_index) + world_index, runtime = await pool.lease() + assert provisioner.reset_calls == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_world_left_unhealthy_by_reset_is_not_handed_out() -> None: + async def scenario() -> None: + pool, provisioner = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}) + await pool.start() + first, _ = await pool.lease() + assert first == 0 # freshly provisioned -- m9 skips this lease's reset + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's (now real) reset hit the scripted UNHEALTHY outcome + await pool.close() + + asyncio.run(scenario()) + + +def test_a_freshly_provisioned_world_failing_its_health_probe_is_not_handed_out() -> None: + # M2: `healthy()` is called unconditionally after reset — including on the m9 fast path, + # which only skips the (expensive) reset call, never the readiness check. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + return runtime.world_index != 0 + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + world_index, _ = await pool.lease() + assert world_index == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_an_unhealthy_world_is_reconciled_back_in_the_background() -> None: + async def scenario() -> None: + pool, provisioner = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}) + await pool.start() + first, _ = await pool.lease() + assert first == 0 + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's reset failed and it fell out of rotation + await asyncio.sleep(0.2) + assert provisioner.provision_calls >= 2 # start() + the background reconcile + # T1: the world must be genuinely leasable again, not merely still counted in pool.size. + recovered_index, recovered_runtime = await pool.lease() + assert recovered_index == 0 + assert recovered_runtime.state is RuntimeState.READY + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_excluding_the_only_world_raises_rather_than_hanging() -> None: + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + world_index, _ = await pool.lease() + await pool.release(world_index) + try: + await asyncio.wait_for(pool.lease(exclude=frozenset({world_index})), timeout=1.0) + except hs.NoWorldsAvailable: + pass + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_can_drop_a_world_that_never_recovers() -> None: + async def scenario() -> None: + calls = {"n": 0} + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return [_runtime(0, RuntimeState.READY), _runtime(1, RuntimeState.READY)] + return [_runtime(1, RuntimeState.READY)] # world 0 degraded away, every time + + async def reset(self, runtime, *, work_directory): + runtime.state = RuntimeState.UNHEALTHY if runtime.world_index == 0 else RuntimeState.READY + + async def healthy(self, runtime, *, work_directory): + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(2, provisioner=Provisioner()) + await pool.start() + first, _ = await pool.lease() + assert first == 0 # freshly provisioned -- m9 skips this lease's reset + await pool.release(0) + world_index, _ = await pool.lease() + assert world_index == 1 # world 0's (now real) reset marked it unhealthy + await asyncio.sleep(0.2) + assert pool.size == 1 # the reconcile's own `provision()` never brings world 0 back + await pool.close() + + asyncio.run(scenario()) + + +def test_concurrent_mark_unhealthy_never_calls_provision_reentrantly() -> None: + # T7/B1: two worlds going bad in the same tick must serialize onto one provider call at a + # time. `overlaps` is asserted here, in the TEST's own frame — an in-band assert inside + # `_serialized` would instead land inside `_reconcile`'s `except Exception` and never fail + # this test. + async def scenario() -> None: + pool, provisioner = _pool(2) + await pool.start() + w0, _ = await pool.lease() + w1, _ = await pool.lease() + await asyncio.gather( + pool.mark_unhealthy(w0, cause="boom0"), pool.mark_unhealthy(w1, cause="boom1") + ) + await asyncio.sleep(0.1) + assert pool.size == 2 + world_index, runtime = await pool.lease() + assert runtime.state is RuntimeState.READY + assert provisioner.overlaps == [] + await pool.close() + assert provisioner.overlaps == [] + + asyncio.run(scenario()) + + +def test_lease_reset_and_a_background_reconcile_never_overlap_on_the_provider() -> None: + # TH-4/B2: the overlap that actually matters is a lease()'s reset() running concurrently + # with a DIFFERENT world's reconcile provision() -- the old coalescer test never drove two + # DIFFERENT provider calls at once. `overlaps` is asserted from the test's own frame: the + # old in-band assert inside `_serialized` fired inside `_reconcile`'s `except Exception` / + # `lease()`'s `except Exception as exc: reset_exc = exc` and was silently swallowed by + # production error-handling, so all three `_provider_lock` sites had zero effective + # coverage. `reset_calls` is pinned too, so the test cannot silently stop driving the + # overlap it claims to. + async def scenario() -> None: + pool, provisioner = _pool(2) + await pool.start() + + # Use up world 1's "fresh" fast path so its NEXT lease pays for a real reset() call -- + # otherwise it would never touch the provider at all and the overlap wouldn't be real. + w1, _ = await pool.lease(exclude=frozenset({0})) + assert w1 == 1 + await pool.release(1) + + w0, _ = await pool.lease(exclude=frozenset({1})) + assert w0 == 0 + + results = await asyncio.gather( + pool.mark_unhealthy(0, cause="boom"), # schedules a background reconcile provision() + pool.lease(exclude=frozenset({0})), # world 1's reset()+healthy() run concurrently + ) + leased = results[1] + assert leased is not None and leased[0] == 1 + await asyncio.sleep(0.1) # let the reconcile finish + assert provisioner.provision_calls >= 2 + assert provisioner.reset_calls >= 1 + assert provisioner.overlaps == [] + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_recovers_the_world_index_when_the_provider_call_races_a_replaced_runtime_object() -> None: + # The R14 discard branch used to drop a replaced-object's index from `_leased` without + # putting it back anywhere -- not `_available`, not `_down`. Every later candidate set then + # stays empty forever and `lease()` hangs. Dead in production today (the real provider + # mutates `EnvironmentRuntime` in place, never replaces it), but the fix has to hold + # independent of that reachability argument, so this drives the replacement directly. + async def scenario() -> None: + holder: dict[str, Any] = {} + + class SwapOnce(FakeProvisioner): + def __init__(self, instances: int) -> None: + super().__init__(instances) + self.armed = False # only swap once the test's OWN lease call is under way + self._swapped = False + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + if self.armed and not self._swapped: + self._swapped = True + # Simulate a concurrent reconcile replacing this index's own + # `EnvironmentRuntime` object between this lease's provider call and its + # post-call re-read. + holder["pool"]._runtimes[runtime.world_index] = _runtime( + runtime.world_index, RuntimeState.READY + ) + return runtime.state is RuntimeState.READY + + provisioner = SwapOnce(1) + pool, _ = _pool(1, provisioner=provisioner) + holder["pool"] = pool + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + + provisioner.armed = True + before = provisioner.healthy_calls + world_index, runtime = await asyncio.wait_for(pool.lease(), timeout=1.0) + assert world_index == 0 + assert runtime.state is RuntimeState.READY + assert provisioner.healthy_calls - before == 2 # the swapped attempt, then the retry that succeeded + await pool.close() + + asyncio.run(scenario()) + + +def test_close_waits_for_an_in_flight_reconcile_before_closing_the_provider() -> None: + # TH-3/R4: `ProcessRuntimeProvider.provision` is `asyncio.to_thread` — cancelling the + # awaiting coroutine does NOT stop the underlying thread. The fake here dispatches through + # `asyncio.to_thread` too (a `threading.Event` gates when the thread-backed work actually + # finishes), so this can only pass if `close()` genuinely waits for the real work instead of + # racing a hard-clean against it. + async def scenario() -> None: + events: list[str] = [] + release_provision = threading.Event() + + def provision_sync(instances: int) -> list[EnvironmentRuntime]: + events.append("provision-start") + release_provision.wait(timeout=5.0) + events.append("provision-end") + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return await asyncio.to_thread(provision_sync, instances) + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return runtime.state is RuntimeState.READY + + async def close(self, *, work_directory): + events.append("close") + + pool, _ = _pool(1, provisioner=Provisioner()) + release_provision.set() + await pool.start() + release_provision.clear() + events.clear() # drop start()'s own provision-start/-end + + await pool.mark_unhealthy(0, cause="boom") # schedules a reconcile mid-flight + await asyncio.sleep(0.05) # let the reconcile's provision() actually begin on its thread + assert events == ["provision-start"] + + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert events == ["provision-start"], "close() ran the provider's own close() too early" + + release_provision.set() # let the thread-backed provision() finish on its own + await asyncio.wait_for(close_task, timeout=5.0) + assert events == ["provision-start", "provision-end", "close"] + + asyncio.run(scenario()) + + +def test_close_during_an_in_flight_reconcile_never_overlaps_the_providers_close_call() -> None: + # §4.5b: the old bounded-wait-then-cancel let close() run `provisioner.close()` CONCURRENTLY + # with a still-live `provision()` once the internal 30s bound expired -- cancelling the + # awaiting coroutine cannot stop underlying thread-backed work. Real `asyncio.to_thread` + # dispatch here (a plain `asyncio.sleep`-backed fake would let even the OLD cancel-then-close + # behavior look correct, since cancellation genuinely stops a sleeping coroutine). `overlaps` + # -- the same state mechanism the reentrancy tests above build -- is the assertion a + # regression back to cancel-then-close would trip. + async def scenario() -> None: + release_provision = threading.Event() + + class SlowThreadedProvisioner(FakeProvisioner): + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + + def _blocking() -> list[EnvironmentRuntime]: + release_provision.wait(timeout=5.0) + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + + runtimes = await asyncio.to_thread(_blocking) + for runtime in runtimes: + self._runtimes[runtime.world_index] = runtime + return runtimes + + provisioner = SlowThreadedProvisioner(1) + release_provision.set() + pool, _ = _pool(1, provisioner=provisioner) + await pool.start() + release_provision.clear() + + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="boom") # schedules a reconcile mid-flight + await asyncio.sleep(0.05) # let the reconcile's provision() actually begin on its thread + + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert not provisioner.closed, "close() ran the provider's own close() before provision() returned" + + release_provision.set() # let the thread-backed provision() finish on its own + await asyncio.wait_for(close_task, timeout=5.0) + assert provisioner.closed is True + assert provisioner.overlaps == [] + + asyncio.run(scenario()) + + +def test_mark_unhealthy_and_lease_after_close_are_blocked() -> None: + # R5: `close()` latches -- neither a late `mark_unhealthy()` (e.g. a scenario's `finally` + # racing a SIGTERM-triggered close()) nor a fresh `lease()` may touch the provider again once + # the pool has been closed, and a second `close()` is a no-op. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + world_index, _ = await pool.lease() + await pool.close() + assert provisioner.provision_calls == 1 # start()'s call only + assert provisioner.closed is True + + await pool.mark_unhealthy(world_index, cause="late failure") + await asyncio.sleep(0.05) + assert provisioner.provision_calls == 1 # no reconcile spawned post-close + + try: + await asyncio.wait_for(pool.lease(), timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + + await pool.close() # second close() is a no-op + assert provisioner.provision_calls == 1 + + asyncio.run(scenario()) + + +def test_close_retried_after_a_callers_own_timeout_still_closes_the_provisioner() -> None: + # A caller wrapping the WHOLE close() call in its own timeout (the entrypoint's + # `_bounded_close`) used to cancel close() after `_closed` had already latched -- + # a retry then hit the old `if self._closed: return` idempotency check and returned + # IMMEDIATELY, claiming completion while teardown could still be running (or, with the check + # removed but no shield, cancelled the shared teardown outright). Either bug shows up as the + # retry NOT genuinely blocking until teardown finishes -- checked directly below, rather than + # via an eventually-true `provisioner.closed` (a to_thread-backed close() leaks its thread on + # real cancellation and can flip that flag on its own regardless of whether close() re-awaited + # it, the same non-cancellable-thread shape the test above covers). + async def scenario() -> None: + events: list[str] = [] + release_close = threading.Event() + + class SlowCloseProvisioner(FakeProvisioner): + async def close(self, *, work_directory: Path) -> None: + async with self._serialized("close"): + + def _blocking() -> None: + release_close.wait(timeout=5.0) + events.append("provider-close-done") + + await asyncio.to_thread(_blocking) + self.closed = True + + provisioner = SlowCloseProvisioner(1) + pool, _ = _pool(1, provisioner=provisioner) + await pool.start() + + # First call: the CALLER's own timeout fires while provisioner.close() is still blocked + # on its thread -- this must not stop the teardown itself (asyncio.shield). + try: + await asyncio.wait_for(pool.close(), timeout=0.1) + except asyncio.TimeoutError: + pass + else: + raise AssertionError("expected the first close() to time out while provisioner.close() blocks") + events.append("first-close-timed-out") + assert provisioner.closed is False # still mid-teardown, not abandoned + + # Retry, started while the provider is STILL blocked (release_close not yet set) -- it + # must genuinely wait, not return claiming completion (the early-return bug) or race an + # independently-cancelled teardown to a coincidentally-correct result. + retry_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + assert not retry_task.done(), "retry close() returned before teardown actually finished" + events.append("retry-still-waiting") + + release_close.set() + await asyncio.wait_for(retry_task, timeout=2.0) + events.append("retry-close-returned") + + assert events == [ + "first-close-timed-out", "retry-still-waiting", "provider-close-done", "retry-close-returned", + ] + assert provisioner.closed is True + assert provisioner.overlaps == [] + + asyncio.run(scenario()) + + +def test_close_blocks_a_reset_that_wins_the_provider_lock_race_after_close_has_latched() -> None: + # `_closed` is set (under `_state_lock`, no `_provider_lock` needed) the moment close() + # starts -- but a lease already past the top-of-loop `_closed` check can still + # win the `_provider_lock` FIFO queue race and call reset() against a provider close() is + # about to hard-clean. Reproduces the race by holding `_provider_lock` externally so both + # close() and a queued lease() are genuinely waiting on it when it releases. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + + await pool._provider_lock.acquire() # stand in for "some provider call already in flight" + lease_task = asyncio.create_task(pool.lease()) + await asyncio.sleep(0.05) # let lease() clear the top `_closed` check and queue on the lock + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) # close() latches `_closed` (no lock needed) and also queues + pool._provider_lock.release() # FIFO: lease() was queued first, so it goes first + + try: + await asyncio.wait_for(lease_task, timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + assert provisioner.reset_calls == 0 # never touched the provider once closed had latched + + await asyncio.wait_for(close_task, timeout=1.0) + + asyncio.run(scenario()) + + +def test_close_blocks_a_healthy_probe_that_wins_the_provider_lock_race_after_close_has_latched() -> None: + # The healthy() block specifically: a freshly-provisioned world skips reset() (m9) and + # goes straight to healthy() -- that site needs the same re-check as the reset block above, + # not just the reset block. + async def scenario() -> None: + pool, provisioner = _pool(1) + await pool.start() # world 0 is "fresh" -- its first lease skips reset(), pays for healthy() + + await pool._provider_lock.acquire() + lease_task = asyncio.create_task(pool.lease()) + await asyncio.sleep(0.05) + close_task = asyncio.create_task(pool.close()) + await asyncio.sleep(0.05) + pool._provider_lock.release() + + try: + await asyncio.wait_for(lease_task, timeout=1.0) + except hs.NoWorldsAvailable as exc: + assert exc.reason == "closed" + else: + raise AssertionError("expected NoWorldsAvailable(reason='closed')") + assert provisioner.healthy_calls == 0 + + await asyncio.wait_for(close_task, timeout=1.0) + + asyncio.run(scenario()) + + +def test_start_degrades_when_provision_returns_fewer_worlds_than_instances() -> None: + # R2: `provision()` legitimately returns fewer worlds than requested (conformance-gate + # failure, `fixed_port`) — spine v1.12 §4: "Fail -> effective parallelism 1 ... + # Loud, never silent," not a `RuntimeError` from this pool. + async def scenario() -> None: + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [_runtime(0)] # only 1 of the 3 requested + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(3, provisioner=Provisioner()) + runtimes = await pool.start() + assert len(runtimes) == 1 + assert pool.effective_size == 1 + world_index, _ = await pool.lease() + assert world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_effective_size_tracks_a_reconcile_not_just_start() -> None: + # `_effective_size` used to be a start()-time snapshot `_reconcile` never touched -- P10 + # sizes `parallelism_degraded` off this property, so a stale value would announce the wrong + # world count outbound. Here start() degrades to 1 of 3, then a reconcile recovers the full + # 3 -- `effective_size` must follow it back up. + async def scenario() -> None: + calls = {"n": 0} + + class Provisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return [_runtime(0, RuntimeState.READY)] # degraded: 1 of 3 requested + return [_runtime(i, RuntimeState.READY) for i in range(instances)] # fully recovered + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + pool, _ = _pool(3, provisioner=Provisioner()) + await pool.start() + assert pool.effective_size == 1 + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="force a reconcile") + await asyncio.sleep(0.1) + assert pool.size == 3 + assert pool.effective_size == 3 + await pool.close() + + asyncio.run(scenario()) + + +def test_start_rejects_a_genuinely_malformed_provision_result() -> None: + # R2: the degrade allowance is not a blanket exemption — zero worlds and a non-contiguous + # index set are still rejected as malformed. + class ZeroWorldsProvisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [] + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + class GapProvisioner: + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + return [_runtime(0), _runtime(2)] # world_index 1 missing -- not contiguous + + async def reset(self, runtime, *, work_directory): + pass + + async def healthy(self, runtime, *, work_directory): + return True + + async def close(self, *, work_directory): + pass + + async def zero_worlds() -> None: + pool, _ = _pool(2, provisioner=ZeroWorldsProvisioner()) + try: + await pool.start() + except RuntimeError: + pass + else: + raise AssertionError("expected RuntimeError for zero worlds") + + async def gap() -> None: + pool, _ = _pool(3, provisioner=GapProvisioner()) + try: + await pool.start() + except RuntimeError: + pass + else: + raise AssertionError("expected RuntimeError for a non-contiguous index set") + + asyncio.run(zero_worlds()) + asyncio.run(gap()) + + +def test_pool_exhaustion_surfaces_a_uniform_never_retried_section_2f_code_from_lease() -> None: + # hosted-execution-seams.md v1.13 §5.4: a deterministic §2f fault (domain environment/agent, + # never retried) used to be discarded at the reset()/reconcile seam and re-reported as + # retryable `world_pool_exhausted`/infrastructure -- burning every whole-job retry on a fault + # that fails identically every time. + # + # Isolated to `lease()`'s OWN reset()/healthy() extraction: `provision()` always SUCCEEDS + # (so the reconcile's separate give-up extraction never runs and cannot backfill the same + # code), and the reconcile's post-provision healthy-probe failing again is what keeps the + # world down -- a first version of this test had `provision()` also fail typed, which masked + # a mutated-away `lease()` extraction because the reconcile's own give-up path recorded the + # same code independently. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + outbound = FakeOutbound() + pool, _ = _pool(1, provisioner=Provisioner(1), outbound=outbound) + await pool.start() + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1, + ) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is not None + assert result.aborted.code == "seed_failed" + assert result.aborted.domain == "environment" + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_surfaces_a_uniform_never_retried_section_2f_code_from_reconcile() -> None: + # The OTHER extraction site, isolated the same way in reverse -- `reset()` fails UNTYPED + # (so `lease()`'s own extraction always yields `None` and cannot backfill), and only the + # reconcile's give-up path ever sees a typed `ProcessRuntimeError`. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise RuntimeError("generic reset failure") # untyped -- lease()'s own code is None + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "seed_failed" + assert exc.domain is hs.FailureDomain.ENVIRONMENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_with_mixed_section_2f_codes_stays_world_pool_exhausted() -> None: + # Mixed codes across the unhealthy worlds must NOT surface either one -- v1.13 only + # promotes a code that is UNIFORM across every currently-unhealthy world. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + code = "seed_failed" if runtime.world_index == 0 else "store_statement_failed" + raise ProcessRuntimeError("reset", code, "boom") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + code = "seed_failed" if runtime.world_index == 0 else "store_statement_failed" + raise ProcessRuntimeError("reset", code, "boom") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + # Untyped -- must not overwrite `_down_codes` with a uniform code. + raise RuntimeError("provider is generically down") + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_with_a_uniform_infrastructure_domain_code_stays_world_pool_exhausted() -> None: + # `store_statement_failed` IS §2f-typed and uniform here, but its domain is + # infrastructure (retryable) -- v1.13 only promotes `environment`/`agent`, so this must still + # fall back to the generic exhaustion abort. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + raise ProcessRuntimeError("reset", "store_statement_failed", "boom") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_pool_exhaustion_uses_the_carried_domain_not_the_fallback_map() -> None: + # v1.15 §2f: the producer resolves `spawn_failed`'s managed/source split AT THE RAISE and + # carries it on `ProcessRuntimeError.domain` -- the scheduler must read that directly. The + # fallback map's own `spawn_failed` entry is `infrastructure` (retryable, never surfaced by + # exhaustion); this raises `spawn_failed` carrying `agent` instead, which IS a never-retried + # domain. Only reading the CARRIED domain makes this surface as `spawn_failed`/`agent` -- + # a scheduler that fell back to the map (or ignored `domain` entirely) would see + # `infrastructure`, which is not in `_SECTION_2F_NEVER_RETRIED`, and this would incorrectly + # stay the generic `world_pool_exhausted` instead. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError( + "reset", "spawn_failed", "agent process exec failed", + domain=hs.FailureDomain.AGENT, + ) + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError( + "reset", "spawn_failed", "agent process exec failed", + domain=hs.FailureDomain.AGENT, + ) + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "spawn_failed" + assert exc.domain is hs.FailureDomain.AGENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_give_up_with_an_untyped_final_attempt_clears_a_stale_typed_code() -> None: + # A world demoted by a typed `seed_failed` reset failure used to keep that code in + # `_down_codes` forever if the reconcile that follows gives up UNTYPED (a bare `OSError`, or + # any non-§2f exception) -- the give-up path only overwrote when its OWN failure was typed, + # so this stale code outlived the attempt that actually produced it and a later exhaustion + # declaration read it as if it were current. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + async with self._serialized(f"reset(w{runtime.world_index})"): + self.reset_calls += 1 + raise ProcessRuntimeError("reset", "seed_failed", "db/seed.sql: exited 1") + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + async with self._serialized("provision"): + self.provision_calls += 1 + if self.provision_calls == 1: + return [_runtime(i, RuntimeState.READY) for i in range(instances)] + # Every reconcile attempt after the initial `start()` fails UNTYPED -- the + # give-up path must clear the `seed_failed` code the reset() failure recorded, + # not leave it standing. + raise OSError("transient enospc") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + first, _ = await pool.lease() + await pool.release(first) # consume the m9 fresh flag -- the next lease pays for reset() + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_lease_exhaustion_considers_every_down_world_not_just_the_exclude_narrowed_subset() -> None: + # A retry lease's `exclude` set narrows `usable` to the runtimes NOT being avoided -- using + # `usable` (rather than every currently-unhealthy world) as the uniformity set let the + # excluded world's own failure escape the check entirely, so a lone untyped down world + # sitting outside `exclude` could hide behind a uniform typed code from everyone else and + # wrongly surface as that code+domain instead of the generic exhaustion abort. + # + # Isolated to the uniformity computation alone: `healthy()` always returns `False` (never + # raises), so the post-provision recovery branch never runs and never pops `_down_codes` -- + # only the two codes this test sets directly via `mark_unhealthy()` are ever in play. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return False + + pool, _ = _pool(2, provisioner=Provisioner(2)) + await pool.start() + await pool.mark_unhealthy(0, cause="generic reset failure", code=None) # untyped + await pool.mark_unhealthy(1, cause="seed_failed", code="seed_failed") # typed, environment + try: + # Excludes world 0 -- a scenario retrying away from the world it just failed on. The + # only OTHER down world (1) carries a uniform typed code on its own, but world 0's own + # untyped failure must still block the pass-through. + await asyncio.wait_for(pool.lease(exclude=frozenset({0})), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_success_with_a_failed_health_probe_clears_a_stale_typed_code() -> None: + # A world demoted with a typed code can be re-provisioned successfully and still fail its + # post-provision health probe -- that probe returning `False` (not raising) is untyped, and + # the give-up path above never runs on this branch because `provision()` itself succeeded. The + # world stays down, but nothing about this round produced a §2f code, so a later exhaustion + # declaration must see the generic `world_pool_exhausted` shape, not the code an earlier, + # superseded demotion happened to record. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + return False # untyped -- provision() itself always succeeds + + pool, provisioner = _pool(1, provisioner=Provisioner(1)) + await pool.start() + await pool.mark_unhealthy(0, cause="seed reset failed", code="seed_failed") # stale typed code + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code is None + assert exc.domain is None + else: + raise AssertionError("expected NoWorldsAvailable") + assert provisioner.provision_calls >= 2 # the reconcile actually re-provisioned + await pool.close() + + asyncio.run(scenario()) + + +def test_reconcile_success_with_a_typed_health_probe_failure_surfaces_that_codes_own_domain() -> None: + # The other direction of the same probe: when the post-provision health check itself raises a + # typed §2f error, that is the round's own result and must replace whatever an earlier, + # superseded demotion recorded -- not merely clear it to `None`. + async def scenario() -> None: + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + async with self._serialized(f"healthy(w{runtime.world_index})"): + self.healthy_calls += 1 + raise ProcessRuntimeError("healthy", "build_failed", "container image missing on retry") + + pool, _ = _pool(1, provisioner=Provisioner(1)) + await pool.start() + await pool.mark_unhealthy(0, cause="seed reset failed", code="seed_failed") # different, stale code + try: + await asyncio.wait_for(pool.lease(), timeout=3.0) + except hs.NoWorldsAvailable as exc: + assert exc.code == "build_failed" + assert exc.domain is hs.FailureDomain.AGENT + else: + raise AssertionError("expected NoWorldsAvailable") + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unhealthy_emitted_exactly_once_per_demotion_path() -> None: + # R6: every demotion path now goes through `WorldPool.mark_unhealthy()`, the sole emitter — + # parametrized over three distinct triggers (the fourth, the `_Retry(mark_unhealthy=True)` + # path, is already covered by `test_world_unavailable_retries_once_on_a_fresh_world_and_recovers` + # and `test_world_unhealthy_cause_is_truncated_and_redacted`). + async def run_scheduler_case(setup_fn: Any, sub_goals: Any) -> list[dict[str, Any]]: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1, + ) + scenarios = [FakeScenario("s1", "id-1", setup_fn=setup_fn, sub_goals=sub_goals)] + await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + await pool.close() + return [kwargs for event, kwargs in outbound.events if event == "world_unhealthy"] + + async def m13_discard_branch() -> None: + # the M13 discard branch: a plain `setup_crashed` fault. + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + events = await run_scheduler_case(boom, [FakeSubGoal("g", lambda w, c: None)]) + assert len(events) == 1 + assert events[0]["world_index"] == 0 + + async def b3_finally_branch() -> None: + # the B3 `finally` branch: something blows past every handled path in `_execute`. + class BrokenSubGoals: + def __bool__(self) -> bool: + raise RuntimeError("sub_goals blew up") + + events = await run_scheduler_case(lambda world: None, BrokenSubGoals()) + assert len(events) == 1 + assert events[0]["world_index"] == 0 + + async def lease_reset_failure_demotion() -> None: + # `WorldPool.lease()`'s own reset/health-probe demotion — pool-level, no scheduler. + outbound = FakeOutbound() + pool, _ = _pool(2, reset_scripts={0: [RuntimeState.UNHEALTHY]}, outbound=outbound) + await pool.start() + first, _ = await pool.lease() + assert first == 0 + await pool.release(0) + await pool.lease() # world 0's real reset hits the scripted UNHEALTHY outcome + events = [kwargs for event, kwargs in outbound.events if event == "world_unhealthy"] + assert len(events) == 1 + assert events[0]["world_index"] == 0 + await pool.close() + + asyncio.run(m13_discard_branch()) + asyncio.run(b3_finally_branch()) + asyncio.run(lease_reset_failure_demotion()) + + +def test_bundle_dir_is_threaded_through_to_every_provision_call() -> None: + # T7/M1: `bundle_dir` must reach both `start()`'s and `_reconcile()`'s `provision()` calls. + async def scenario() -> None: + seen: list[Path] = [] + + class Provisioner(FakeProvisioner): + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + seen.append(bundle_dir) + return await super().provision( + bundle, source=source, bundle_dir=bundle_dir, work_directory=work_directory, + contract=contract, instances=instances, + ) + + bundle_dir = Path("/work/bundle-xyz") + pool = hs.WorldPool( + Provisioner(1), bundle=object(), source=Path("/work/source"), bundle_dir=bundle_dir, + work_directory=Path("/work"), instances=1, + ) + await pool.start() + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="force a reconcile") + await asyncio.sleep(0.1) + assert seen and all(path == bundle_dir for path in seen) + await pool.close() + + asyncio.run(scenario()) + + +# --- HostedScheduler --------------------------------------------------------------------------- + + +def test_two_scenarios_pass_over_two_worlds() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, provisioner = _pool(2, outbound=outbound) + await pool.start() + call = hs.Call(name="book", arguments={}, ok=True) + runner = FakeCallRunner({"s1": _call_outcome(calls=(call,)), "s2": _call_outcome(calls=(call,))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=100, + ) + scenarios = [ + FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]), + ] + result = await scheduler.run(scenarios) + assert result.aborted is None + assert {r.status for r in result.receipts} == {"passed"} + assert all(r.scenario_attempt == 1 for r in result.receipts) + assert {e for e, _ in outbound.events} == {"scenario_started"} + assert provisioner.provision_calls == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_not_held_check_is_failed_not_errored() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: "the combo was never ordered")])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "failed" + assert receipt.sub_goals[0] == hs.SubGoalResult(name="goal", held=False, reason="the combo was never ordered", judged=False) + assert receipt.failure is None + await pool.close() + + asyncio.run(scenario()) + + +def test_ready_not_ready_errors_without_retrying() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", ready_fn=lambda w: "precondition missing", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure == hs.ReceiptFailure(domain="simulator", stage="running", code="ready_not_ready", message="precondition missing") + assert receipt.scenario_attempt == 1 + assert receipt.sub_goals[0].held is None + assert runner.calls == [] # never reached the call step + # M13: `ready_not_ready` is a clean verdict, not an exception -- the world is released, + # not discarded, so it is immediately leasable again. + world_index, _ = await pool.lease() + assert world_index == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_setup_crash_errors_without_retrying() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=boom, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "setup_crashed" + assert receipt.scenario_attempt == 1 + # M13: an exception outcome discards the world (mark_unhealthy), never release() -- it is + # down, not immediately available, until the background reconcile recovers it. + assert 0 in pool._down + await pool.close() + + asyncio.run(scenario()) + + +def test_world_usage_misuse_in_ready_maps_to_world_usage() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + def misuse(world: Any) -> None: + raise WorldUsageError("cannot invent a table") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", ready_fn=misuse, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "world_usage" + assert receipt.failure.domain == "simulator" + assert 0 in pool._down # M13: discarded, not released + await pool.close() + + asyncio.run(scenario()) + + +def test_state_too_large_from_check_errors_without_retrying() -> None: + # T7: `state_too_large` was never exercised by the original suite. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + + def blow_up(world: Any, calls: Any) -> None: + raise WorldStateTooLarge("table 'events' exceeds the baseline cap") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", blow_up)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "state_too_large" + assert receipt.failure.domain == "simulator" + assert receipt.scenario_attempt == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_a_phase_over_its_budget_times_out() -> None: + # T4: real scenario code is synchronous — a `time.sleep` body is what actually exercises + # B4's dedicated-executor dispatch. The old `async def` + `asyncio.sleep` version would have + # passed even with B4 unfixed (`wait_for` can always interrupt a coroutine's own suspension + # point), so it never caught the bug. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.05 + try: + def slow(world: Any) -> None: + time.sleep(1.0) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure.code == "setup_timeout" + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await pool.close() + + asyncio.run(scenario()) + + +def test_an_async_phase_over_its_budget_still_times_out() -> None: + # B4: "keep the awaitable branch" — async scenario code (reached via a sync wrapper, as + # `FakeScenario.setup` always is) must still be interruptible on budget. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.05 + try: + async def slow(world: Any) -> None: + await asyncio.sleep(1.0) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure.code == "setup_timeout" + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await pool.close() + + asyncio.run(scenario()) + + +def test_concurrent_worlds_make_progress_while_one_is_blocked_in_sync_code() -> None: + # B4/TH-1: BOTH worlds' setup sleeps -- serialized execution would take >=0.4s, concurrent + # execution ~0.2s. The old version slept only world 0, so serialized and parallel wall times + # were indistinguishable (both ~0.2s) and the elapsed-time assertion alone could not catch a + # regression to serial dispatch; `started_order` now pins the actual interleaving too. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + started_order: list[str] = [] + + def both_slow(world: Any) -> None: + started_order.append(f"start-{world.world_index}") + time.sleep(0.2) + started_order.append(f"end-{world.world_index}") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=both_slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=both_slow, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + t0 = asyncio.get_running_loop().time() + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + elapsed = asyncio.get_running_loop().time() - t0 + assert all(r.status == "passed" for r in result.receipts) + assert elapsed < 0.3 # serialized would be >=0.4s + # Both worlds must have STARTED before either one ENDED -- genuine concurrency, not just + # a fast total. + assert set(started_order[:2]) == {"start-0", "start-1"}, started_order + await pool.close() + + asyncio.run(scenario()) + + +def test_a_leaked_phase_thread_does_not_starve_a_sibling_world_and_close_still_completes() -> None: + # R1 (highest-ranked missing test): a phase whose thread never returns must not stop the + # NEXT scenario's phase from running, and `pool.close()` must still complete promptly. With + # the shared default executor this used to fail once enough threads leaked; with a dedicated + # `ThreadPoolExecutor` (`shutdown(wait=False, cancel_futures=True)` on `run()` exit) both hold. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.1 + try: + def maybe_runaway(world: Any) -> None: + if world.world_index == 0: + time.sleep(5.0) # abandoned -- this thread never returns + + runner = FakeCallRunner({"s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=maybe_runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=maybe_runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].failure is not None and result.receipts[0].failure.code == "setup_timeout" + assert result.receipts[1].status == "passed" # world 1's phase ran despite world 0's leak + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await asyncio.wait_for(pool.close(), timeout=2.0) # must not hang behind the leaked thread + + asyncio.run(scenario()) + + +def test_scenario_phases_run_on_the_dedicated_hosted_scenario_executor() -> None: + # R1's actual claim is WHOSE executor phase threads run on, not merely how many workers it + # has -- reverting `loop.run_in_executor(executor, _run)` back to + # `asyncio.to_thread(_run)` (the loop's shared default executor) survived every test in the + # suite, because nothing pinned the dispatch target itself. Capture the real thread name from + # inside a phase body. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + seen_thread_name = {"name": ""} + + def capture(world: Any) -> None: + seen_thread_name["name"] = threading.current_thread().name + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", setup_fn=capture, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "passed" + assert seen_thread_name["name"].startswith("hosted-scenario"), seen_thread_name["name"] + await pool.close() + + asyncio.run(scenario()) + + +def test_executor_sizing_covers_more_scenarios_than_the_leak_headroom_alone() -> None: + # `_LEAK_HEADROOM` alone assumes spine §1's `scenario_count` admission cap (<=10) -- W=1, N + # scenarios that all abandon their setup thread, N > effective_size + _LEAK_HEADROOM (11). + # Before the fix, scenarios past the headroom found the executor saturated and were reported + # `driver_crashed` for a phase that never even started, instead of the genuine + # `setup_timeout` every one of them actually is. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + original = hs.SETUP_TIMEOUT_SECONDS + hs.SETUP_TIMEOUT_SECONDS = 0.1 + try: + def runaway(world: Any) -> None: + time.sleep(5.0) # abandoned -- never returns within the test + + n = 13 # > effective_size(1) + _LEAK_HEADROOM(10) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario(f"s{i}", f"id-{i}", setup_fn=runaway, sub_goals=[FakeSubGoal("g", lambda w, c: None)]) + for i in range(n) + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=15.0) + codes = [r.failure.code for r in result.receipts if r.failure is not None] + assert codes.count("driver_crashed") == 0, codes + assert codes.count("setup_timeout") == n + finally: + hs.SETUP_TIMEOUT_SECONDS = original + await asyncio.wait_for(pool.close(), timeout=2.0) + + asyncio.run(scenario()) + + +def test_check_broken_leaves_later_subgoals_unjudged() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[ + FakeSubGoal("goal1", lambda w, c: None), + FakeSubGoal("goal2", lambda w, c: 42), # wrong return type -> broken + FakeSubGoal("goal3", lambda w, c: None), + ])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "check_broken" + assert [g.held for g in receipt.sub_goals] == [True, None, None] + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_declared_sub_goals_is_check_broken_not_a_vacuous_pass() -> None: + # m7: `all(())` is vacuously True — must not read as a silent pass. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "check_broken" + assert receipt.sub_goals == () + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unavailable_retries_once_on_a_fresh_world_and_recovers() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, provisioner = _pool(2, outbound=outbound) + await pool.start() + attempts = {"n": 0} + + def check(world: Any, calls: Any) -> None: + attempts["n"] += 1 + if attempts["n"] == 1: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_started") == 2 + assert kinds.count("scenario_retried") == 1 + assert kinds.count("world_unhealthy") == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unhealthy_cause_is_truncated_and_redacted() -> None: + # T7/M7: `cause` is capped at 200 chars and must never carry endpoint credentials. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + leaky_message = "connection failed: postgresql://harness:s3cr3t@localhost:14000/w0 " + ("x" * 300) + + def blow_up(world: Any, calls: Any) -> None: + raise WorldUnavailable(leaky_message) + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", blow_up)])] + await scheduler.run(scenarios) + causes = [kwargs["cause"] for event, kwargs in outbound.events if event == "world_unhealthy"] + assert causes + cause = causes[0] + assert len(cause) <= 200 + assert "s3cr3t" not in cause + assert "://***@" in cause + await pool.close() + + asyncio.run(scenario()) + + +def test_mark_unhealthy_schedules_recovery_before_the_telemetry_emit() -> None: + # `_schedule_reconcile()` used to run AFTER the `world_unhealthy` emit -- a slow (or hanging) + # `OutboundPort` call delayed recovery by exactly its own duration, and worst case (an emit + # that never returns) `_reconcile_pending` stays latched forever and `lease()`'s grace loop + # spins without ever declaring exhaustion. + async def scenario() -> None: + emit_started = asyncio.Event() + emit_release = asyncio.Event() + + class SlowOutbound(FakeOutbound): + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + emit_started.set() + await emit_release.wait() + await super().world_unhealthy(world_index=world_index, cause=cause) + + outbound = SlowOutbound() + pool, provisioner = _pool(1, outbound=outbound) + await pool.start() + world_index, _ = await pool.lease() + + mark_task = asyncio.create_task(pool.mark_unhealthy(world_index, cause="boom")) + await asyncio.wait_for(emit_started.wait(), timeout=1.0) + # The emit is blocked -- recovery must already have been scheduled by this point. + assert pool._reconcile_task is not None, "reconcile was not scheduled ahead of the emit" + + emit_release.set() + await asyncio.wait_for(mark_task, timeout=1.0) + await asyncio.sleep(0.05) + await pool.close() + + asyncio.run(scenario()) + + +def test_world_unavailable_twice_gives_up_after_the_one_retry() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + def always_fails(world: Any, calls: Any) -> None: + raise WorldUnavailable("always broken") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", always_fails)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.scenario_attempt == 2 + assert receipt.failure.code == "world_unavailable" + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_between_attempt_1_and_attempt_2_reports_errored_not_skipped() -> None: + # R3: attempt 1 ran and produced a real errored outcome; a cancel/abort landing on the + # retry-lease path — the "abandoned while queued for the retry world" site — must not lose + # it to skipped-synthesis. outbound-channels.md defines `skipped` as "never ran," and this + # scenario manifestly did. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + cancel_flag = {"v": False} + + def check(world: Any, calls: Any) -> None: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + + real_lease = pool.lease + lease_calls = {"n": 0} + + async def flaky_lease(*, exclude=frozenset(), abandon=None): + lease_calls["n"] += 1 + if lease_calls["n"] == 2: # the retry-lease call, right after attempt 1 failed + cancel_flag["v"] = True + return await real_lease(exclude=exclude, abandon=abandon) + + pool.lease = flaky_lease # type: ignore[method-assign] + + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "world_unavailable" + assert receipt.scenario_attempt == 1 + assert receipt.world_index == 0 + # Defensive, not discriminating on its own: this site's own retry-lease `None` return + # already short-circuits BEFORE the emit both before and after the `scenario_retried` + # placement change (the mutation run confirmed it), so this only pins that the property + # continues to hold -- the actual discriminating test is + # `test_cancel_during_the_retry_leases_own_health_probe...` below, the one whose + # mutation-reverted shape this assertion actually catches. + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_retried") == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_during_the_retry_leases_own_health_probe_reports_errored_not_skipped() -> None: + # R3, the SECOND post-attempt-1 `return None` site: a cancel/abort can also land AFTER the + # retry-lease has already granted a world (during ITS OWN reset/healthy await), rather than + # while queued for one -- a distinct code path from the test above, reached at the top of the + # attempt-2 recursive call instead of inside the retry-lease call itself. + async def scenario() -> None: + cancel_flag = {"v": False} + + class Provisioner(FakeProvisioner): + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + if runtime.world_index == 1: + cancel_flag["v"] = True + async with self._serialized(f"healthy(w{runtime.world_index})"): + return runtime.state is RuntimeState.READY + + outbound = FakeOutbound() + pool, _ = _pool(2, provisioner=Provisioner(2), outbound=outbound) + await pool.start() + + def check(world: Any, calls: Any) -> None: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", check)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "world_unavailable" + assert receipt.scenario_attempt == 1 + assert receipt.world_index == 0 + # The retry lease itself succeeded (world 1 granted), but attempt 2's own cancel + # re-check bailed before it ever proceeded -- `scenario_retried` sits right next to + # attempt 2's `scenario_started` now, so neither fires here either. + kinds = [event for event, _ in outbound.events] + assert kinds.count("scenario_retried") == 0 + await pool.close() + + asyncio.run(scenario()) + + +def test_evidence_missing_retries_without_marking_the_world_unhealthy() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + seen = {"n": 0} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + seen["n"] += 1 + if seen["n"] == 1: + return _call_outcome(turns=1, calls=()) + return _call_outcome(turns=1, calls=(hs.Call(name="x", arguments={}),)) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + kinds = [event for event, _ in outbound.events] + assert kinds.count("world_unhealthy") == 0 # not a world-health problem + assert kinds.count("scenario_retried") == 1 + await pool.close() + + asyncio.run(scenario()) + + +def test_evidence_missing_twice_errors() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + return _call_outcome(turns=1, calls=()) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "evidence_missing" + assert receipt.scenario_attempt == 2 + assert receipt.call is not None # the call step DID run; only evidence capture failed + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_turns_and_zero_calls_is_still_evidence_missing() -> None: + # M10: unconditioned on `turns` — a 0-turn call is an unobserved agent, same as any other + # empty-calls outcome; `evidence_missing` must fire regardless. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + return _call_outcome(turns=0, calls=()) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "evidence_missing" + assert receipt.scenario_attempt == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_aborted_with_partial_evidence_is_reported_not_null() -> None: + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) # `call_failed` is infrastructure-domain (retryable) — needs a spare world + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + raise hs.CallAborted( + "livekit room dropped", + partial=hs.CallOutcome( + calls=(), turns=0, started_at="2026-08-25T00:00:00.000Z", + ended_at="2026-08-25T00:00:01.000Z", duration_ms=1000, + ), + ) + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure.code == "call_failed" + assert receipt.call is not None and receipt.call.duration_ms == 1000 + assert receipt.scenario_attempt == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_aborted_with_no_partial_evidence_still_retries() -> None: + # T6/M3: `retry=call is not None` used to suppress the retry whenever `partial` was `None` — + # world-handle-interface.md v3.3 pins `call_failed` as unconditionally retried once. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class Runner: + def __init__(self) -> None: + self.attempts = 0 + + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + self.attempts += 1 + if self.attempts == 1: + raise hs.CallAborted("livekit room never opened", partial=None) + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + runner = Runner() + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.status == "passed" + assert receipt.scenario_attempt == 2 + assert runner.attempts == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_call_runner_raising_a_bare_exception_maps_to_call_failed() -> None: + # T7/B3: a crash from the call runner that is not a `CallAborted` it deliberately raised is + # the same world-handle-interface.md v3.3 row — "the simulated-call machinery crashed". + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s1": ConnectionError("livekit socket reset")}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + receipt = result.receipts[0] + assert receipt.failure.code == "call_failed" + assert receipt.failure.domain == "infrastructure" + assert receipt.scenario_attempt == 2 + assert receipt.call is None + await pool.close() + + asyncio.run(scenario()) + + +def test_a_bug_in_the_driver_itself_becomes_a_driver_crashed_receipt_without_killing_the_run() -> None: + # T7/B3/R7: a crash that blows past every handled path in `_execute` (not the agent, not a + # check, not the call) must land as `driver_crashed` and must not suppress the OTHER + # scenario's receipt. R7: it must also report the REAL world_index the crash happened on + # (world 0, genuinely leased) rather than always `None`. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenSubGoals: + def __bool__(self) -> bool: + raise RuntimeError("sub_goals blew up") + + runner = FakeCallRunner({ + "s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + }) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + broken = FakeScenario("s1", "id-1") + broken.sub_goals = BrokenSubGoals() # type: ignore[assignment] + scenarios = [broken, FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "errored" + assert result.receipts[0].failure.code == "driver_crashed" + assert result.receipts[0].failure.domain == "simulator" + assert result.receipts[0].world_index == 0 # R7: the real world, not always None + assert result.receipts[0].scenario_attempt == 1 + assert result.receipts[1].status == "passed" # the crash did not suppress this receipt + assert len(outbound.receipts) == 2 + await pool.close() + + asyncio.run(scenario()) + + +def test_driver_crashed_reports_unjudged_sub_goals_when_they_are_readable() -> None: + # R7: `driver_crashed` must carry every declared sub-goal (`held: null`) when `sub_goals` + # itself is perfectly readable — only a crash reading `sub_goals` falls back to `()`. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenReadOnly: + world_index = 0 + + def read_only(self) -> Any: + raise RuntimeError("world.read_only() blew up") + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return BrokenReadOnly() + + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=FakeCallRunner({}), outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g1", lambda w, c: None), FakeSubGoal("g2", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.world_index == 0 + assert receipt.scenario_attempt == 1 + assert [g.name for g in receipt.sub_goals] == ["g1", "g2"] + assert all(g.held is None and g.reason is None for g in receipt.sub_goals) + await pool.close() + + asyncio.run(scenario()) + + +def test_driver_crashed_reports_the_call_summary_when_the_call_step_already_ran() -> None: + # A crash AFTER a successful call step -- here, `world.read_only()` blowing up while building + # the check-phase handle -- must not report `call: null`. The call demonstrably ran; + # outbound-channels.md Channel 2's errored-receipt body only allows `null` when the call + # never started. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + + class BrokenSecondReadOnly: + def __init__(self) -> None: + self.calls = 0 + + def read_only(self) -> Any: + self.calls += 1 + if self.calls == 1: + return object() # used by the default no-op ready_fn; never touched + raise RuntimeError("check-phase read_only() blew up") + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return BrokenSecondReadOnly() + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.call is not None + assert receipt.call.duration_ms == 5000 # _call_outcome()'s fixture value -- the call ran + await pool.close() + + asyncio.run(scenario()) + + +def test_a_retrys_driver_crashed_receipt_does_not_carry_the_previous_attempts_call_summary() -> None: + # `_ScenarioContext` is shared across both attempts of a retry (`_run_scenario` recurses + # with the same `context` object) -- `world_index`/`attempt` were refreshed on entry but + # `call` was not, so an attempt-1 outcome that reached the call step + # (e.g. `evidence_missing`, retried onto a fresh world) left attempt 1's `CallSummary` sitting + # on the context for attempt 2 to inherit if attempt 2 crashed before making its own call. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + class WorldForAttempt: + def __init__(self, world_index: int) -> None: + self.world_index = world_index + self.rng = None + + def read_only(self) -> Any: + if self.world_index == 1: + # attempt 2's own world -- crashes before its call step ever runs. + raise RuntimeError("attempt 2's read_only() blew up before its own call step") + return object() # attempt 1's world -- used by the default no-op ready_fn + + class Factory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> Any: + return WorldForAttempt(runtime.world_index) + + # Attempt 1's call genuinely ran (started_at/duration_ms distinct from any fixture default) + # but captured zero tool calls -- `evidence_missing`, retried onto a fresh world. + runner = FakeCallRunner( + {"s1": hs.CallOutcome(calls=(), turns=1, started_at="A1", ended_at="A1-end", duration_ms=1111)} + ) + scheduler = hs.HostedScheduler(pool=pool, world_factory=Factory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None and receipt.failure.code == "driver_crashed" + assert receipt.scenario_attempt == 2 + assert receipt.world_index == 1 + assert receipt.call is None # attempt 2 never reached its own call step + await pool.close() + + asyncio.run(scenario()) + + +def test_exactly_one_receipt_per_scenario_key_even_when_one_scenario_crashes() -> None: + # T7/B3: `gather(return_exceptions=True)` + the try/finally around the leased region must + # never produce zero or duplicate receipts for any scenario. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + + def boom(world: Any) -> None: + raise ValueError("scenario code bug") + + runner = FakeCallRunner({"s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=boom, sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await scheduler.run(scenarios) + assert [r.status for r in result.receipts] == ["errored", "passed"] + keys = [r.scenario_key for r in outbound.receipts] + assert sorted(keys) == ["s1", "s2"] + assert len(keys) == len(set(keys)) + await pool.close() + + asyncio.run(scenario()) + + +def test_outbound_failures_never_kill_the_run_or_change_the_receipt() -> None: + # B3: a completely broken OutboundPort is best-effort telemetry — never fatal, never + # receipt-affecting. + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + outbound = FailingOutbound(fail_on={"scenario_started", "receipt"}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("goal", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.receipts[0].status == "passed" + assert result.aborted is None + await pool.close() + + asyncio.run(scenario()) + + +def test_a_fence_stops_the_run_from_launching_further_scenarios() -> None: + # `HostedFencedError`/`HostedChannelFailedError` used to be swallowed as best-effort telemetry + # by `_emit`/`_log`/`mark_unhealthy`'s emit, so a superseded attempt ran its ENTIRE scenario + # set and billed a whole attempt's worth of simulated calls after the platform had already + # fenced it (outbound-channels.md: 401/403 -> "stop emitting, exit code 3 ... never an infra + # retry"). A fence on the first scenario's own + # `scenario_started` emit must stop every later scenario from ever launching. + async def scenario() -> None: + class FencingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + if scenario_key == "s0": + raise HostedFencedError( + ChannelError(ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + await super().scenario_started( + scenario_key=scenario_key, world_index=world_index, scenario_attempt=scenario_attempt + ) + + outbound = FencingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + n = 5 + runner = FakeCallRunner({f"s{i}": _call_outcome(calls=(hs.Call(name="x", arguments={}),)) for i in range(n)}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(n)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert isinstance(result.fenced, HostedFencedError) + # s0's fence lands before the call step; if the run had kept launching, s1-s4 would have + # reached the call step too (their own `scenario_started` succeeds). + assert runner.calls == [] + assert outbound.receipts == [] # a superseded attempt emits no receipts + await pool.close() + + asyncio.run(scenario()) + + +def test_a_fence_landing_before_the_world_is_resolved_releases_it_instead_of_demoting_it() -> None: + # A `_FATAL_OUTBOUND` escaping through the first `scenario_started` emit reached + # `_run_scenario`'s `finally` with `world_resolved` still `False`, and `mark_unhealthy()` + # there demoted a world that never did anything wrong -- a + # false `world_unhealthy` emitted after the run had already stopped emitting, and a + # `_schedule_reconcile()` call that spent a `provision()` attempt on a job that already stopped. + async def scenario() -> None: + class FencingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + raise HostedFencedError( + ChannelError(ChannelOutcome.FENCED, None, "fence_mismatch", "attempt superseded") + ) + + outbound = FencingOutbound() + pool, provisioner = _pool(1, outbound=outbound) + await pool.start() + runner = FakeCallRunner({"s0": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario("s0", "id-0", sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert [event for event, _ in outbound.events if event == "world_unhealthy"] == [] + # If a reconcile were (wrongly) scheduled, close() waits for it to finish before + # returning -- checking `provision_calls` only after close() makes this deterministic. + await pool.close() + assert provisioner.provision_calls == 1 # only start()'s own call + + asyncio.run(scenario()) + + +def test_a_channel_failed_error_latches_the_same_fenced_path() -> None: + # The same latch, reached through `WorldPool.mark_unhealthy`'s own emit rather than the + # scheduler's `_emit` -- `HostedChannelFailedError` (404x3, "finalize platform_sync") is just + # as fatal as a fence and must stop the run the same way. + async def scenario() -> None: + class ChannelFailingOutbound(FakeOutbound): + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + raise HostedChannelFailedError( + ChannelError(ChannelOutcome.CHANNEL_FAILED, None, "not_found", "channel gone") + ) + + outbound = ChannelFailingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + world_index, _ = await pool.lease() + await pool.mark_unhealthy(world_index, cause="boom") + assert pool.fenced is not None + assert isinstance(pool.fenced, HostedChannelFailedError) + await pool.close() + + asyncio.run(scenario()) + + +def test_an_attempt_superseded_error_latches_the_same_fenced_path() -> None: + # `HostedAttemptSupersededError` (409 `attempt_superseded`) is the third member of + # outbound.py's `ChannelState` "stop emitting" latch, not just the two originally caught -- + # it is "a fence in substance" per that module's own docstring. Without it in + # `_FATAL_OUTBOUND`, a superseded attempt took the best-effort branch instead and drained its + # entire scenario list against a channel that refuses every request. + async def scenario() -> None: + class SupersedingOutbound(FakeOutbound): + async def scenario_started( + self, *, scenario_key: str, world_index: int, scenario_attempt: int + ) -> None: + if scenario_key == "s0": + raise HostedAttemptSupersededError( + ChannelError(ChannelOutcome.PERMANENT_ITEM, None, "attempt_superseded", "attempt was superseded") + ) + await super().scenario_started( + scenario_key=scenario_key, world_index=world_index, scenario_attempt=scenario_attempt + ) + + outbound = SupersedingOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + n = 5 + runner = FakeCallRunner({f"s{i}": _call_outcome(calls=(hs.Call(name="x", arguments={}),)) for i in range(n)}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(n)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.fenced is not None + assert isinstance(result.fenced, HostedAttemptSupersededError) + assert runner.calls == [] + assert outbound.receipts == [] + await pool.close() + + asyncio.run(scenario()) + + +def test_cancel_after_the_first_scenario_skips_the_rest() -> None: + # T3/TH-2: the original test's `cancel_requested=lambda: True` was true before `run()` was + # even called, so nothing ever launched and the "in-flight scenario finishes" behavior was + # never exercised. Here the flag flips mid-run, from inside the first scenario's own call + # step. TH-2: re-pins the exact six-field `skipped` receipt body outbound-channels.md calls + # "exact" — the assertion that would have caught R3. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(1, outbound=outbound) + await pool.start() + cancel_flag = {"v": False} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + cancel_flag["v"] = True + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + scheduler = hs.HostedScheduler( + pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, + job_seed=1, cancel_requested=lambda: cancel_flag["v"], + ) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("g", lambda w, c: None)]) for i in range(3)] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is None # a cancel is not a job-level failure + assert result.receipts[0].status == "passed" # the in-flight scenario finished + assert result.receipts[1] == hs.ResultReceipt( + scenario_key="s1", scenario_id="id-1", scenario_attempt=1, world_index=None, + status="skipped", sub_goals=(), evaluations=(), call=None, failure=None, + ) + assert result.receipts[2] == hs.ResultReceipt( + scenario_key="s2", scenario_id="id-2", scenario_attempt=1, world_index=None, + status="skipped", sub_goals=(), evaluations=(), call=None, failure=None, + ) + # (outbound-channels.md v1.3 Sequencing: "terminal event -> skipped receipts -> + # manifest"): `run()` only SYNTHESIZES the skipped receipts -- emitting them is the + # caller's job, done after its own terminal event. Only the real "s0" receipt should be + # on the wire at this point. + assert [r.scenario_key for r in outbound.receipts] == ["s0"] + await scheduler.emit_skipped_receipts(result) + assert [r.scenario_key for r in outbound.receipts] == ["s0", "s1", "s2"] + assert outbound.receipts[1] == result.receipts[1] + assert outbound.receipts[2] == result.receipts[2] + await pool.close() + + asyncio.run(scenario()) + + +def test_zero_ready_worlds_aborts_the_run_and_skips_the_rest() -> None: + # T2: the original test raced two workers that both hit `NoWorldsAvailable` independently at + # t~=0 (W=1, both past the pre-check before either leased) — "the rest gets skipped because + # the job aborted" was never actually exercised. Here W=1, scenario 1 genuinely runs (and + # fails on its only world), the provisioner then goes permanently down, and scenarios 2/3 are + # asserted to have never reached the call step at all. + async def scenario() -> None: + provisioner = FakeProvisioner(1) + calls = {"n": 0} + real_provision = provisioner.provision + + async def flaky_provision(bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + calls["n"] += 1 + if calls["n"] == 1: + return await real_provision( + bundle, source=source, bundle_dir=bundle_dir, work_directory=work_directory, + contract=contract, instances=instances, + ) + raise RuntimeError("provisioner is down") + + provisioner.provision = flaky_provision # type: ignore[method-assign] + outbound = FakeOutbound() + pool, _ = _pool(1, provisioner=provisioner, outbound=outbound) + await pool.start() + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + if scenario.scenario_key == "s1": + raise WorldUnavailable("world 0's schema is gone") + raise AssertionError(f"{scenario.scenario_key} should never have reached the call step") + + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [ + FakeScenario("s1", "id-1", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s3", "id-3", sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=5.0) + assert result.aborted is not None + assert result.aborted.domain == "infrastructure" + assert result.aborted.code == "world_pool_exhausted" + statuses = {r.scenario_key: r.status for r in result.receipts} + assert statuses["s1"] == "errored" # M8: ran and failed -- must not read as "never ran" + assert statuses["s2"] == "skipped" + assert statuses["s3"] == "skipped" + started = [event for event, _ in outbound.events if event == "scenario_started"] + assert len(started) == 1 # s2/s3 were genuinely never launched + # Same ordering guarantee -- only the real "s1" receipt is on the wire until the + # caller explicitly asks for the synthesized `skipped` ones. + assert [r.scenario_key for r in outbound.receipts] == ["s1"] + await scheduler.emit_skipped_receipts(result) + skipped_keys = sorted(r.scenario_key for r in outbound.receipts if r.status == "skipped") + assert skipped_keys == ["s2", "s3"] + + asyncio.run(scenario()) + + +def test_pool_size_caps_concurrent_scenario_execution() -> None: + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + concurrency = {"now": 0, "max": 0} + + class Runner: + async def run(self, scenario: FakeScenario, runtime: EnvironmentRuntime) -> hs.CallOutcome: + concurrency["now"] += 1 + concurrency["max"] = max(concurrency["max"], concurrency["now"]) + await asyncio.sleep(0.02) + concurrency["now"] -= 1 + return _call_outcome(calls=(hs.Call(name="x", arguments={}),)) + + outbound = FakeOutbound() + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=Runner(), outbound=outbound, job_seed=1) + scenarios = [FakeScenario(f"s{i}", f"id-{i}", sub_goals=[FakeSubGoal("goal", lambda w, c: None)]) for i in range(4)] + result = await scheduler.run(scenarios) + assert concurrency["max"] == 1 + assert all(r.status == "passed" for r in result.receipts) + await pool.close() + + asyncio.run(scenario()) + + +def test_scenario_seed_is_job_seed_plus_index() -> None: + # T5: one scenario at index 0 cannot distinguish `+index` from `+0` — two scenarios can. + async def scenario() -> None: + pool, _ = _pool(1) + await pool.start() + seen_first_draw: dict[str, int] = {} + + def make_setup(key: str): + def setup(world: Any) -> None: + seen_first_draw[key] = world.rng.randint(0, 10**9) + return setup + + outbound = FakeOutbound() + runner = FakeCallRunner({ + "s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + "s2": _call_outcome(calls=(hs.Call(name="x", arguments={}),)), + }) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=777) + scenarios = [ + FakeScenario("s1", "id-1", setup_fn=make_setup("s1"), sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + FakeScenario("s2", "id-2", setup_fn=make_setup("s2"), sub_goals=[FakeSubGoal("g", lambda w, c: None)]), + ] + await scheduler.run(scenarios) + assert seen_first_draw["s1"] == random.Random(777 + 0).randint(0, 10**9) + assert seen_first_draw["s2"] == random.Random(777 + 1).randint(0, 10**9) + await pool.close() + + asyncio.run(scenario()) + + +def test_a_retry_reseeds_the_rng_identically() -> None: + # world-handle-interface.md Determinism: "a retry re-seeds identically" — attempt 2 must draw + # the SAME first value as attempt 1 did (both `Random(job.seed + scenario_index)`), not + # continue attempt 1's stream and not use a different seed. + async def scenario() -> None: + outbound = FakeOutbound() + pool, _ = _pool(2, outbound=outbound) + await pool.start() + draws: list[int] = [] + attempts = {"n": 0} + + def setup(world: Any) -> None: + draws.append(world.rng.randint(0, 10**9)) + attempts["n"] += 1 + if attempts["n"] == 1: + raise WorldUnavailable("world 0 lost its schema") + + runner = FakeCallRunner({"s1": _call_outcome(calls=(hs.Call(name="x", arguments={}),))}) + scheduler = hs.HostedScheduler(pool=pool, world_factory=FakeWorldFactory(), call_runner=runner, outbound=outbound, job_seed=555) + scenarios = [FakeScenario("s1", "id-1", setup_fn=setup, sub_goals=[FakeSubGoal("g", lambda w, c: None)])] + result = await scheduler.run(scenarios) + assert result.receipts[0].status == "passed" + assert result.receipts[0].scenario_attempt == 2 + assert len(draws) == 2 + expected = random.Random(555 + 0).randint(0, 10**9) + assert draws[0] == expected + assert draws[1] == expected + await pool.close() + + asyncio.run(scenario()) diff --git a/tests/harness/test_outbound.py b/tests/harness/test_outbound.py new file mode 100644 index 00000000..cfd62c22 --- /dev/null +++ b/tests/harness/test_outbound.py @@ -0,0 +1,2961 @@ +"""Outbound reporting, per `outbound-channels.md` v1.2. Two parts, matching `outbound.py`: + +Foundations (part 1): + +- Canonicalization: `canonical_bytes`/`event_payload_digest`/`whole_object_digest` against fixed + expected digests, independently derived by running the contract's own literal + ``json.dumps(..., sort_keys=True, separators=(",", ":"), ensure_ascii=False)`` + sha256 call in + a bare Python shell -- not by calling back into this module -- so the vectors don't just prove + the implementation agrees with itself. +- Capabilities: `HostedCapabilities`/`load_capabilities` against the contract's own example, + its endpoint-trailing-slash rule, schema-version gate, and the load-then-unlink lifetime rule. +- Events + spool: `build_event_record`/`HostedEvent` shape validation for all nine closed event + types, and `OutboundSpool`'s contiguous-from-1 sequencing, its crash/torn-tail recovery rule + (a synthetic partial write, simulating a crash mid-append), and watermark semantics. + +Transport (part 2): + +- Pure functions: the full `classify_response` error map (every status the contract's "Error + responses"/"Failure semantics summary" names, plus the unlisted-4xx catch-all), the full-jitter + `compute_backoff_seconds` formula, `format_rfc3339_millis`, and `ArtifactBudgetTracker`. +- Typed models: `ResultReceiptDraft` (contract-shaped, the `skipped`/`errored` exact-shape rules, + digest tamper rejection) and `ArtifactManifestDraft` (digest scope, `complete` distinguishing + documents). +- `FakePlatform`: an in-process, no-sockets `Transport` implementation with real server-side state + (accepted events + watermark, receipts keyed by `(job_id, scenario_key)`, artifacts keyed by + digest, manifests keyed by `(attempt_id, digest)`) that `EventsClient`/`ResultsClient`/ + `ArtifactsClient` are driven against for: success + watermark advance, transient-then-success + (a queued `TransportError` then a queued 5xx then real success), 429 honoring `Retry-After` over + the computed backoff, deterministic rejection (never retried, at both the per-event and + whole-request level), `FENCED`/`CHANNEL_FAILED` raising the two channel-ending exceptions, the + artifact digest-mismatch "re-upload once" rule, budget refusal, chunked-transfer reassembly, and + crash-between-send-and-ack redelivery (the platform durably records a write and then the + response is lost -- `FakePlatform.crash_after_next_write` -- proving a redelivery of the same + record is a safe no-op, both within one client call via its own retry loop and across two + independent calls simulating a real process restart). +""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import logging +import os +import threading +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest +from pydantic import ValidationError + +from fi.alk.harness.job import FailureDomain, HarnessStage +from fi.alk.harness.outbound import ( + CAPABILITIES_SCHEMA_VERSION, + EVENT_PAYLOAD_MAX_BYTES, + ArtifactBudgetTracker, + ArtifactKind, + ArtifactManifestDraft, + ArtifactsClient, + CapabilitiesError, + ChannelOutcome, + ChannelState, + EventsClient, + HostedAttemptSupersededError, + HostedCapabilities, + HostedChannelFailedError, + HostedEndpoints, + HostedEvent, + HostedEventDraft, + HostedFencedError, + OutboundError, + OutboundEventType, + OutboundSpool, + OutboundSpoolError, + ResultReceiptDraft, + ResultsClient, + RetryPolicy, + ScenarioStatus, + StageChangedPayload, + TransportError, + TransportResponse, + build_artifact_manifest, + build_event_record, + build_result_receipt, + build_skipped_receipt, + canonical_bytes, + classify_response, + compute_backoff_seconds, + event_payload_digest, + format_rfc3339_millis, + is_valid_digest, + is_valid_rfc3339_millis, + load_capabilities, + priority_class, + redact_outbound_text, + sha256_digest, + truncate_log_message, + whole_object_digest, +) + +NOW = datetime(2026, 8, 25, 10, 14, 3, 412000, tzinfo=timezone.utc) +LATER = datetime(2026, 8, 25, 10, 17, 7, 623000, tzinfo=timezone.utc) + + +@pytest.fixture(autouse=True) +def _clear_spool_registry() -> Any: + """M6's in-process registry caches `OutboundSpool` instances (and their open flock fds) for the + life of the interpreter, keyed on `(resolved_root, name)`. Different tests use different + `tmp_path` roots so there is no cross-test key collision, but without this the fds would still + accumulate across the whole session. Runs after every test.""" + yield + OutboundSpool._clear_registry_for_tests() + + +# --- Canonicalization ------------------------------------------------------------------------- + + +def test_canonical_bytes_sorts_keys_and_uses_compact_separators() -> None: + assert canonical_bytes({"b": 1, "a": 2}) == b'{"a":2,"b":1}' + + +def test_canonical_bytes_does_not_escape_non_ascii() -> None: + # ensure_ascii=False: a non-ASCII character is emitted as its raw UTF-8 bytes, not a \uXXXX + # escape -- the contract is explicit about the exact `json.dumps` call, and this is the one + # kwarg that differs from Python's default. + encoded = canonical_bytes({"x": "…"}) + assert encoded == b'{"x":"\xe2\x80\xa6"}' + assert b"\\u" not in encoded + + +def test_canonical_bytes_absent_and_null_are_different_bytes() -> None: + assert canonical_bytes({}) != canonical_bytes({"x": None}) + assert canonical_bytes({"x": None}) == b'{"x":null}' + + +def test_event_payload_digest_matches_independently_computed_vector() -> None: + # The Channel 1 example payload from outbound-channels.md, transcribed verbatim (including its + # literal "…" ellipsis character, which also exercises the ensure_ascii=False rule end-to-end). + payload = {"scenario_key": "…", "world_index": 2, "scenario_attempt": 1} + digest = event_payload_digest(payload) + # Computed independently via `python -c "import json,hashlib; ..."` against the contract's own + # literal algorithm text, not by calling back into this module. + assert digest == ( + "sha256:7a8037965edd1f5391e6d59d0556313938abbd272ca162cbf75aa1510ca7e0f1" + ) + assert is_valid_digest(digest) + + +def test_event_payload_digest_scope_is_payload_only_not_the_envelope() -> None: + payload = {"world_index": 2, "cause": "boom"} + envelope = {"event_id": "event_x", "stage": "running", "type": "world_unhealthy", "payload": payload} + # The envelope's other fields must not affect the digest -- only `payload` is in scope. + assert event_payload_digest(payload) == event_payload_digest(envelope["payload"]) + assert sha256_digest(canonical_bytes(payload)) == event_payload_digest(payload) + + +def test_whole_object_digest_scope_excludes_the_digest_key_by_popping_it() -> None: + core = {"a": 1, "b": 2} + with_digest = {"a": 1, "digest": "sha256:" + "f" * 64, "b": 2} + # "the whole object with the `digest` key absent" -- popped, so the digest's own value (right + # or wrong, present or not) can never affect the hash the platform re-derives and checks. + assert whole_object_digest(core) == whole_object_digest(with_digest) + + +def test_whole_object_digest_matches_receipt_and_manifest_shaped_examples() -> None: + # Channel 2's example receipt object, minus its own `digest` field (as if the guest is about + # to compute it) -- proves the generic function against a receipt-shaped whole object, not + # just the toy `{a, b}` case above. + receipt = { + "schema_version": "futureagi.harness-result.v1", + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "scenario_key": "suspended-account-blocked", + "scenario_id": "sid-1", + "scenario_attempt": 1, + "world_index": 2, + "status": "passed", + "sub_goals": [{"name": "n", "held": True, "reason": None, "judged": False}], + "evaluations": [], + "call": None, + "failure": None, + } + digest = whole_object_digest(receipt) + assert is_valid_digest(digest) + # FIXED expected hex (ranked missing test 2): computed independently via + # `json.dumps(receipt, sort_keys=True, separators=(",", ":"), ensure_ascii=False, + # allow_nan=False)` + sha256 in a bare Python shell against the contract's own canonicalization + # clause -- not by calling back into this module. This is the digest the platform ACTUALLY + # verifies (event digests are only MAY-verified; receipt/manifest digests are verified), so a + # self-consistency-only test (is_valid_digest, invariance to `digest`, inequality between two + # objects) cannot catch a canonicalization regression that both this test and the module agree + # on by construction -- a fixed vector can. + assert digest == "sha256:b1492bcb2f62c3d79dc4d0d1b3f5097a3784112fc00276a2dc483297375652af" + # Adding a `digest` key of any value must not change the result (scope excludes it). + assert whole_object_digest({**receipt, "digest": "sha256:" + "0" * 64}) == digest + + # Channel 3b's manifest example, minus `digest`. + manifest = { + "schema_version": "futureagi.harness-manifest.v1", + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "entries": [{"artifact_id": "sha256:" + "1" * 64, "kind": "result", "size": 10, "scenario_key": "k"}], + "complete": True, + } + manifest_digest = whole_object_digest(manifest) + assert is_valid_digest(manifest_digest) + # FIXED expected hex, same independent-derivation method as above. + assert manifest_digest == "sha256:f18786749321d103cf30ebe11e935bf9177d5297783ee2dec00cb20ad8dae13d" + assert manifest_digest != digest + + +def test_canonical_bytes_rejects_nan() -> None: + # v1.3: allow_nan=False -- NaN/Infinity are not RFC 8259 and must fail typed at the emitter + # rather than produce bytes no strict parser downstream can read. + with pytest.raises(OutboundError) as excinfo: + canonical_bytes({"score": float("nan")}) + assert excinfo.value.code == "canonical_value_not_finite" + + +@pytest.mark.parametrize("value", [float("inf"), float("-inf")]) +def test_canonical_bytes_rejects_infinity(value: float) -> None: + with pytest.raises(OutboundError): + canonical_bytes({"score": value}) + + +def test_canonical_bytes_is_still_byte_identical_for_valid_input_after_allow_nan_false() -> None: + # "for valid input the bytes are identical" (v1.3) -- allow_nan=False changes nothing about + # ordinary values. + assert canonical_bytes({"b": 1, "a": 2}) == b'{"a":2,"b":1}' + + +def test_whole_object_digest_rejects_a_non_json_native_value_naming_the_key_path() -> None: + with pytest.raises(OutboundError) as excinfo: + whole_object_digest({"a": {"b": datetime(2026, 1, 1, tzinfo=timezone.utc)}}) + assert excinfo.value.code == "digest_value_not_json_native" + assert "a.b" in excinfo.value.message + + +def test_whole_object_digest_rejects_a_non_string_dict_key() -> None: + with pytest.raises(OutboundError): + whole_object_digest({"a": {1: "x"}}) + + +# --- Capabilities ------------------------------------------------------------------------------ + +_ATTEMPT_ID = "22222222-2222-2222-2222-222222222222" + +VALID_CAPABILITIES = { + "schema_version": CAPABILITIES_SCHEMA_VERSION, + "job_id": "11111111-1111-1111-1111-111111111111", + "attempt_id": _ATTEMPT_ID, + "attempt_number": 1, + "fence": "opaque-fence", + # Far future: load_capabilities rejects an expired token against wall-clock + # time, so a near-term timestamp turns the whole fixture into a time bomb. + "expires_at": "2099-01-01T00:00:00.000Z", + "token": "bearer-token", + "endpoints": { + "events": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/events/", + "results": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/results/", + "artifacts": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/artifacts/", + "scenarios": f"https://platform.example/simulate/api/harness/attempts/{_ATTEMPT_ID}/scenarios/", + }, +} + + +def _write(path: Path, value: dict) -> Path: + path.write_text(json.dumps(value), encoding="utf-8") + return path + + +def test_load_capabilities_parses_the_contract_example_and_builds_auth_headers(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + assert capabilities.endpoints.events.endswith("/") + assert capabilities.auth_headers() == { + "Authorization": "Bearer bearer-token", + "X-Harness-Fence": "opaque-fence", + } + + +def test_load_capabilities_unlinks_the_file_after_a_successful_load(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + load_capabilities(path) # default unlink=True + assert not path.exists() + + +def test_load_capabilities_does_not_unlink_on_a_validation_failure(tmp_path: Path) -> None: + bad = {**VALID_CAPABILITIES, "schema_version": "futureagi.harness-capabilities.v0"} + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError): + load_capabilities(path) + # A crash mid-load must never destroy the only copy of a not-yet-consumed bearer. + assert path.exists() + + +def test_load_capabilities_rejects_missing_file(tmp_path: Path) -> None: + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(tmp_path / "missing.json") + assert excinfo.value.code == "capabilities_file_missing" + + +def test_load_capabilities_rejects_unreadable_file(tmp_path: Path) -> None: + if os.geteuid() == 0: + pytest.skip("permission bits don't block a root reader") + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + path.chmod(0o000) + try: + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_unreadable" + finally: + path.chmod(0o600) + + +def test_load_capabilities_rejects_malformed_json(tmp_path: Path) -> None: + path = tmp_path / "capabilities.json" + path.write_text("{not json", encoding="utf-8") + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_malformed" + + +def test_load_capabilities_rejects_valid_json_that_is_not_an_object(tmp_path: Path) -> None: + path = tmp_path / "capabilities.json" + path.write_text("[1, 2, 3]", encoding="utf-8") + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_file_malformed" + + +def test_load_capabilities_rejects_wrong_schema_version(tmp_path: Path) -> None: + bad = {**VALID_CAPABILITIES, "schema_version": "futureagi.harness-capabilities.v0"} + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + # Lens 5: this code used to be unreachable -- pydantic wrapped it into a ValidationError that + # only ever surfaced as the generic capabilities_field_invalid. + assert excinfo.value.code == "capabilities_schema_unsupported" + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_an_endpoint_missing_its_trailing_slash( + tmp_path: Path, channel: str +) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].rstrip("/") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + # Also previously unreachable as a code -- same dead-code shape as the schema check above. + assert excinfo.value.code == "capabilities_endpoint_invalid" + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_a_non_https_endpoint(tmp_path: Path, channel: str) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].replace("https://", "http://") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_endpoint_insecure" + + +def test_load_capabilities_rejects_an_expired_token(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path, now=lambda: datetime(2100, 1, 1, tzinfo=timezone.utc)) + assert excinfo.value.code == "capabilities_expired" + assert path.exists() # a failed load must never unlink -- same rule as any other rejection + + +def test_load_capabilities_accepts_a_token_not_yet_expired(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, now=lambda: datetime(2020, 1, 1, tzinfo=timezone.utc)) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts", "scenarios"]) +def test_load_capabilities_rejects_an_endpoint_whose_attempt_id_segment_disagrees( + tmp_path: Path, channel: str +) -> None: + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["endpoints"][channel] = bad["endpoints"][channel].replace(_ATTEMPT_ID, "some-other-attempt") + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_attempt_mismatch" + + +def test_load_capabilities_field_invalid_message_never_echoes_the_input_value(tmp_path: Path) -> None: + # MIN-3: the file carries the bearer -- a validation message built from pydantic's default + # str(exc) would embed the failing field's raw value. + bad = json.loads(json.dumps(VALID_CAPABILITIES)) + bad["token"] = 987654321 + path = _write(tmp_path / "capabilities.json", bad) + with pytest.raises(CapabilitiesError) as excinfo: + load_capabilities(path) + assert excinfo.value.code == "capabilities_field_invalid" + assert "987654321" not in excinfo.value.message + + +def test_load_capabilities_unlink_failure_is_reported_via_the_callback(tmp_path: Path) -> None: + # MIN-7: previously swallowed unconditionally; now the caller can tell. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + original_unlink = Path.unlink + + def failing_unlink(self: Path, *a: Any, **kw: Any) -> None: + if self == path: + raise OSError("simulated unlink failure") + return original_unlink(self, *a, **kw) + + reported: list[OSError] = [] + Path.unlink = failing_unlink # type: ignore[method-assign] + try: + load_capabilities(path, on_unlink_failure=reported.append) + finally: + Path.unlink = original_unlink # type: ignore[method-assign] + assert len(reported) == 1 + assert path.exists() # the failed unlink really did leave the file behind + + +def test_load_capabilities_insecure_file_mode_warns_but_does_not_block_the_load( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + # MIN-5: fail-safe -- a wrong mode/owner is a loud warning, never a rejection. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + path.chmod(0o644) + with caplog.at_level("WARNING"): + capabilities = load_capabilities(path, unlink=False) + assert capabilities.job_id == VALID_CAPABILITIES["job_id"] + assert any("capabilities file mode" in record.message for record in caplog.records) + + +def test_hosted_endpoints_rejects_unknown_field() -> None: + with pytest.raises(ValidationError): + HostedCapabilities.model_validate({**VALID_CAPABILITIES, "unexpected": "field"}) + + +def test_hosted_endpoints_shape_validator_still_fires_on_direct_construction() -> None: + # Defense-in-depth: constructing the model directly (bypassing load_capabilities' + # pre-pydantic checks entirely) must still reject a malformed endpoint. + with pytest.raises(ValidationError): + HostedEndpoints.model_validate( + {"events": "http://x/", "results": "https://x/", "artifacts": "https://x/", "scenarios": "https://x/"} + ) + + +def test_capabilities_expires_at_is_serialized_as_millis_z() -> None: + capabilities = HostedCapabilities.model_validate( + {**VALID_CAPABILITIES, "expires_at": "2026-08-25T12:00:00.123456+00:00"} + ) + assert capabilities.model_dump(mode="json")["expires_at"] == "2026-08-25T12:00:00.123Z" + + +def test_capabilities_event_builder_binds_identity_fields_from_the_token(tmp_path: Path) -> None: + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + build = capabilities.event_builder() + record = build( + event_id="event_" + "a" * 32, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + assert record["job_id"] == capabilities.job_id + assert record["attempt_id"] == capabilities.attempt_id + assert record["attempt_number"] == capabilities.attempt_number + + +def test_capabilities_event_builder_forwards_extra_secret_values_to_log_message(tmp_path: Path) -> None: + # P4: `event_builder()` had no way to receive the job's declared secret VALUES, so a caller + # wired for it (the shipped adapter binds identity via `event_builder()` and routes every event + # through it) could never satisfy N9's redaction requirement for `log.message`/ + # `terminal.failure.message` through this API -- only the userinfo scrub reached those fields. + path = _write(tmp_path / "capabilities.json", VALID_CAPABILITIES) + capabilities = load_capabilities(path, unlink=False) + build = capabilities.event_builder(extra_secret_values=("s3cr3t-value",)) + record = build( + event_id="event_" + "a" * 32, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "error", "message": "build failed: s3cr3t-value leaked in the output"}, + ) + assert "s3cr3t-value" not in record["payload"]["message"] + assert "***" in record["payload"]["message"] + + +# --- Events -------------------------------------------------------------------------------- + + +def _event(type_: OutboundEventType, stage: HarnessStage, payload: dict) -> dict: + return build_event_record( + event_id="event_" + "a" * 32, + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=stage, + type=type_, + payload=payload, + ) + + +def test_build_event_record_has_no_sequence_key_and_embeds_a_valid_digest() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "hello"} + ) + assert "sequence" not in record + assert is_valid_digest(record["digest"]) + assert record["digest"] == event_payload_digest(record["payload"]) + + +def test_hosted_event_requires_sequence_but_draft_does_not() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "debug", "message": "m"} + ) + HostedEventDraft.model_validate(record) # no sequence needed + with pytest.raises(ValidationError): + HostedEvent.model_validate(record) # sequence is required on the full wire object + HostedEvent.model_validate({**record, "sequence": 1}) + + +@pytest.mark.parametrize( + ("type_", "stage", "payload"), + [ + ( + OutboundEventType.STAGE_CHANGED, + HarnessStage.RUNNING, + {"from": "connecting_agent", "to": "running"}, + ), + (OutboundEventType.STAGE_CHANGED, HarnessStage.QUEUED, {"from": None, "to": "queued"}), + ( + OutboundEventType.PARALLELISM_DEGRADED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"requested": 4, "effective": 1, "reason": "conformance_gate_failed"}, + ), + ( + OutboundEventType.BASELINE_FROZEN, + HarnessStage.VALIDATING_ENVIRONMENT, + {"inputs_digest": "sha256:" + "1" * 64, "baseline_ref": "build-1"}, + ), + ( + OutboundEventType.BASELINE_INPUTS_CHANGED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"previous_digest": None, "current_digest": "sha256:" + "2" * 64}, + ), + ( + OutboundEventType.WORLD_UNHEALTHY, + HarnessStage.RUNNING, + {"world_index": 1, "cause": "sentinel failed"}, + ), + ( + OutboundEventType.SCENARIO_STARTED, + HarnessStage.RUNNING, + {"scenario_key": "k", "world_index": 0, "scenario_attempt": 1}, + ), + ( + OutboundEventType.SCENARIO_RETRIED, + HarnessStage.RUNNING, + {"scenario_key": "k", "from_world": 0, "to_world": 1}, + ), + (OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "warning", "message": "m"}), + ( + OutboundEventType.TERMINAL, + HarnessStage.COMPLETED, + { + "stage": "completed", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 2, "failed": 0, "errored": 0, "skipped": 0}, + }, + ), + ( + OutboundEventType.TERMINAL, + HarnessStage.CANCELED, + { + "stage": "canceled", + "reason": "user_canceled", + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 1}, + }, + ), + ], +) +def test_every_closed_event_type_accepts_its_contract_shaped_payload( + type_: OutboundEventType, stage: HarnessStage, payload: dict +) -> None: + record = _event(type_, stage, payload) + HostedEvent.model_validate({**record, "sequence": 1}) + + +def test_stage_changed_to_must_equal_the_events_own_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.STAGE_CHANGED, + HarnessStage.RUNNING, + {"from": None, "to": "grading"}, + ) + + +def test_stage_changed_rejects_the_python_attribute_name_instead_of_the_wire_alias() -> None: + # M9: populate_by_name=True previously let a caller who writes `from_stage` (the natural + # mistake -- that's the attribute name) pass validation while spooling an undefined wire key. + with pytest.raises(ValidationError): + StageChangedPayload.model_validate({"from_stage": "queued", "to": "running"}) + + +def test_terminal_payload_stage_must_equal_the_events_own_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.TERMINAL, + HarnessStage.COMPLETED, + { + "stage": "failed", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + + +def test_terminal_payload_rejects_a_non_terminal_stage() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.TERMINAL, + HarnessStage.RUNNING, + { + "stage": "running", + "reason": None, + "failure": None, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + + +def test_scenario_started_scenario_attempt_is_closed_to_one_or_two() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.SCENARIO_STARTED, + HarnessStage.RUNNING, + {"scenario_key": "k", "world_index": 0, "scenario_attempt": 3}, + ) + + +def test_parallelism_degraded_effective_must_be_strictly_below_requested() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.PARALLELISM_DEGRADED, + HarnessStage.VALIDATING_ENVIRONMENT, + {"requested": 3, "effective": 3, "reason": "fixed_port"}, + ) + + +def test_unknown_event_type_is_rejected_by_the_closed_vocabulary() -> None: + with pytest.raises(ValidationError): + build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type="not_a_real_type", # type: ignore[arg-type] + payload={}, + ) + + +def test_payload_with_an_unknown_key_is_rejected() -> None: + with pytest.raises(ValidationError): + _event( + OutboundEventType.LOG, + HarnessStage.RUNNING, + {"level": "info", "message": "m", "extra": "nope"}, + ) + + +def test_oversized_payload_for_a_non_log_type_is_still_hard_rejected() -> None: + # M8: only `log` is truncated instead of rejected -- every other type keeps the hard rejection. + # `baseline_frozen` has no field-level max_length, so an artificially huge `baseline_ref` is a + # clean way to exercise the envelope-level size check for a non-log type. + with pytest.raises(ValidationError): + _event( + OutboundEventType.BASELINE_FROZEN, + HarnessStage.VALIDATING_ENVIRONMENT, + {"inputs_digest": "sha256:" + "1" * 64, "baseline_ref": "x" * (EVENT_PAYLOAD_MAX_BYTES + 1)}, + ) + + +def test_log_payload_is_truncated_rather_than_rejected_when_oversized() -> None: + huge_message = "x" * (EVENT_PAYLOAD_MAX_BYTES * 2) + record = _event(OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "error", "message": huge_message}) + assert len(canonical_bytes(record["payload"])) <= EVENT_PAYLOAD_MAX_BYTES + assert record["payload"]["message"].endswith("…[truncated]") + # The digest must match the TRUNCATED bytes actually spooled, not the original message. + assert record["digest"] == event_payload_digest(record["payload"]) + HostedEvent.model_validate({**record, "sequence": 1}) # round-trips cleanly + + +def test_truncate_log_message_is_a_no_op_when_already_within_budget() -> None: + assert truncate_log_message("info", "short message") == "short message" + + +def test_truncate_log_message_fits_the_exact_byte_budget() -> None: + message = truncate_log_message("error", "y" * 1000, max_payload_bytes=64) + assert message.endswith("…[truncated]") + assert len(canonical_bytes({"level": "error", "message": message})) <= 64 + + +def test_a_tampered_digest_is_rejected() -> None: + record = _event( + OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "m"} + ) + record["digest"] = "sha256:" + "0" * 64 + with pytest.raises(ValidationError): + HostedEventDraft.model_validate(record) + + +def test_event_id_is_checked_against_utf8_byte_length_not_just_character_count() -> None: + # N-2: max_length=64 on the model counts characters; the platform's column is presumably + # bytes. 64 non-ASCII characters can exceed 64 bytes while passing the character-count check. + event_id = "€" * 64 # 64 chars, 3 bytes each in UTF-8 => 192 bytes + with pytest.raises(ValidationError): + build_event_record( + event_id=event_id, + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + + +def test_hosted_event_emitted_at_is_serialized_as_millis_z_not_the_pydantic_default() -> None: + record = _event(OutboundEventType.LOG, HarnessStage.RUNNING, {"level": "info", "message": "m"}) + assert record["emitted_at"] == "2026-08-25T10:14:03.412Z" + + +def test_hosted_event_emitted_at_rejects_a_naive_datetime() -> None: + with pytest.raises(ValidationError): + build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=datetime(2026, 8, 25, 10, 0, 0), # naive + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + + +def test_hosted_event_emitted_at_normalizes_a_non_utc_offset_to_z() -> None: + ist = datetime(2026, 8, 25, 15, 44, 3, 412000, tzinfo=timezone(timedelta(hours=5, minutes=30))) + record = build_event_record( + event_id="event_x", + job_id="job-1", + attempt_id="attempt-1", + attempt_number=1, + emitted_at=ist, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": "m"}, + ) + assert record["emitted_at"] == "2026-08-25T10:14:03.412Z" + + +# --- Spool --------------------------------------------------------------------------------- + + +def test_spool_assigns_contiguous_sequence_numbers_starting_at_one(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + sequences = [spool.append({"event_id": f"e{i}"}).sequence for i in range(5)] + assert sequences == [1, 2, 3, 4, 5] + assert spool.next_sequence == 6 + + +def test_spool_appended_body_round_trips_the_assigned_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + record = spool.append({"event_id": "e1", "payload": {"x": 1}}) + decoded = record.decode() + assert decoded["sequence"] == 1 + assert decoded["event_id"] == "e1" + + +def test_spool_recovers_next_sequence_across_a_clean_restart(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(3): + first.append({"event_id": f"e{i}"}) + + # M6: the same (root, name) returns the CACHED instance within one process -- simulating a + # real restart (a fresh interpreter, an empty registry) needs an explicit evict. + OutboundSpool._forget_for_tests(tmp_path, "events") + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second is not first + assert second.next_sequence == 4 + assert second.append({"event_id": "e3"}).sequence == 4 + + +def test_spool_recovers_from_a_torn_tail_by_truncating_and_reusing_the_sequence( + tmp_path: Path, +) -> None: + """Simulates a crash mid-write: a partial line with no trailing newline is appended directly + to the spool file (bypassing `append`, the way a killed process would leave the file). The + recovery rule this module documents says the next spool instance must detect this, truncate + the file back to the last complete record, and reuse the torn record's sequence number on its + next append -- never skip past it and leave a gap.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + + path = tmp_path / "events.spool.jsonl" + size_before = path.stat().st_size + with path.open("ab") as stream: + stream.write(b'{"event_id":"e3","sequ') # torn: no closing brace, no trailing newline + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 3 + assert path.stat().st_size == size_before # torn bytes truncated away + + replay = recovered.append({"event_id": "e3-retry"}) + assert replay.sequence == 3 + + ids_in_order = [record.decode()["event_id"] for record in recovered.records()] + assert ids_in_order == ["e1", "e2", "e3-retry"] + + +def test_spool_recovers_from_a_torn_tail_with_no_trailing_newline_at_all(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + # Drop the final newline entirely, simulating a crash after the JSON bytes landed but before + # the trailing separator (and its fsync) did. + data = path.read_bytes() + assert data.endswith(b"\n") + path.write_bytes(data[:-1]) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 1 + assert recovered.records() == [] + assert recovered.append({"event_id": "e1-retry"}).sequence == 1 + + +def test_spool_recovery_degrades_on_mid_file_corruption_instead_of_raising(tmp_path: Path) -> None: + """N8 (supersedes the old B1-era "raises" posture): a bad line with COMPLETE records after it + (a trailing `\\n` of its own) is genuine corruption, never a torn tail -- B1 already forbids + silently renumbering past it, but raising unconditionally was the OTHER extreme: it discarded + even the records BEFORE the corruption, permanently, including a future terminal event. Recovery + must instead succeed, read the readable prefix, and flag the corruption once.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + spool.append({"event_id": "e3"}) + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[1] = b"{not valid json but has a trailing newline" # complete line, unparseable + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) # must NOT raise + assert recovered.is_corrupt + assert recovered.corruption_offset is not None + assert [r.decode()["event_id"] for r in recovered.records()] == ["e1"] + # Never renumbers past the corruption: e1 was sequence 1, so the next allocation is 2 -- even + # though the file still physically contains e2/e3 at sequences 2/3 beyond the corrupt line. + assert recovered.next_sequence == 2 + + +def test_spool_recovery_ships_a_record_appended_after_the_corruption_via_the_seek_path( + tmp_path: Path, +) -> None: + """N8, ranked missing test 5: the readable prefix is still usable AND the stream keeps working + going forward -- a new record appended after recovery (standing in for "the terminal event + still ships") is reachable via `pending_since_watermark()`'s seek path (what `EventsClient.flush` + actually uses), even though a full `records()` scan still stops at the untouched corrupt byte.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.append({"event_id": "e2"}) + spool.append({"event_id": "e3"}) + spool.advance_watermark(1) # e1 already acked before the corruption is ever discovered + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[1] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + assert recovered.next_sequence == 2 # only e1 (pre-corruption) counted; watermark=1 agrees + + terminal = recovered.append({"event_id": "terminal"}) + assert terminal.sequence == 2 + pending = recovered.pending_since_watermark() + assert [r.decode()["event_id"] for r in pending] == ["terminal"] + + +def test_spool_compact_through_refuses_on_a_corrupt_spool_instead_of_destroying_records( + tmp_path: Path, +) -> None: + # P1, second victim: `_rewrite_retaining` (what BOTH `compact_through` and `drop_many` funnel + # through) used to rewrite the file from ONLY the records-before-corruption prefix, so a rewrite + # on a corrupt spool silently destroyed every intact record past the corrupt byte too -- + # regardless of `compact_through`'s own watermark clamp, which only bounds WHICH sequences it + # asks to keep, not what the underlying rewrite can actually see. Must now refuse entirely + # rather than lose an already-durable, never-sent record this way. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[0] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + + terminal = recovered.append({"event_id": "event_terminal"}) + recovered.advance_watermark(terminal.sequence) # fully acked -- would normally be compactable + recovered.compact_through(terminal.sequence) + + assert b"event_terminal" in path.read_bytes(), ( + "P1 regression: compact_through destroyed a record on a corrupt spool" + ) + + +def test_spool_records_degrades_on_corruption_instead_of_raising( + tmp_path: Path, +) -> None: + # N8: records() shares _iter_complete_records with _recover -- corruption discovered via a + # fresh read (not at construction time) degrades the same way: the readable prefix is still + # returned, never a bare exception (nor the old typed raise) a caller has no reason to catch. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + with path.open("ab") as stream: + stream.write(b"{not valid json}\n") # complete (trailing \n), unparseable + assert not spool.is_corrupt + result = spool.records() + assert [r.decode()["event_id"] for r in result] == ["e1"] + assert spool.is_corrupt + assert spool.corruption_offset is not None + + +def test_spool_corruption_is_logged_once_not_once_per_read(tmp_path: Path, caplog: Any) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + path = tmp_path / "events.spool.jsonl" + with path.open("ab") as stream: + stream.write(b"{not valid json}\n") + with caplog.at_level(logging.ERROR, logger="fi.alk.harness.outbound"): + spool.records() + spool.records() + spool.records() + corruption_logs = [r for r in caplog.records if "spool corrupt" in r.message] + assert len(corruption_logs) == 1 + + +def test_spool_failed_append_is_a_true_no_op(tmp_path: Path) -> None: + """B1 part 1: a write that raises mid-flush must leave the file exactly as it was before the + attempt (truncated back to size_before, fsync'd) so the retry reuses the same sequence number + on clean ground -- never merges torn bytes with the retry's complete record into one + unparseable line.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "ok1"}) + path = tmp_path / "events.spool.jsonl" + size_before = path.stat().st_size + + class Boom(Exception): + pass + + original_open = Path.open + + def failing_open(self: Path, mode: str = "r", *a: Any, **kw: Any) -> Any: + if self == path and mode == "ab": + raise Boom("simulated write failure") + return original_open(self, mode, *a, **kw) + + Path.open = failing_open # type: ignore[method-assign] + try: + with pytest.raises(Boom): + spool.append({"event_id": "boom"}) + finally: + Path.open = original_open # type: ignore[method-assign] + + assert path.stat().st_size == size_before + + replay = spool.append({"event_id": "ok2"}) + assert replay.sequence == 2 # reused, not skipped + assert [r.decode()["event_id"] for r in spool.records()] == ["ok1", "ok2"] + + +def test_spool_fsyncs_the_directory_once_after_first_file_creation_only(tmp_path: Path) -> None: + # M2: a new file's directory entry isn't durable just because the file's own data is fsync'd. + # The guard flag means exactly ONE extra fsync (the directory's) on the append that creates the + # file, and none on later appends to the same, now-existing file. + calls: list[int] = [] + original_fsync = os.fsync + + def counting_fsync(fd: int) -> None: + calls.append(fd) + original_fsync(fd) + + os.fsync = counting_fsync # type: ignore[assignment] + try: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + before = len(calls) + spool.append({"event_id": "e1"}) + after_first = len(calls) + spool.append({"event_id": "e2"}) + after_second = len(calls) + finally: + os.fsync = original_fsync # type: ignore[assignment] + + assert after_first - before == 2 # the record's own fsync + the one-time directory fsync + assert after_second - after_first == 1 # just the record's own fsync -- no repeat directory sync + + +def test_canonical_bytes_never_contains_a_raw_newline_even_with_embedded_newlines_in_a_string() -> ( + None +): + # The recovery rule's line-per-record framing depends on this holding for every value this + # module ever canonicalizes: json.dumps escapes control characters inside strings, so an + # embedded "\n" in a field value becomes the two-byte escape `\\n`, never a raw newline byte. + # `OutboundSpool.append` asserts this invariant defensively (`outbound_spool_record_unframable`) + # but the assertion is unreachable through this function -- there is no dict this module can be + # asked to canonicalize that trips it. + assert b"\n" not in canonical_bytes({"message": "line one\nline two"}) + + +def test_spool_watermark_starts_at_zero_and_advances(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) + assert spool.watermark() == 0 + assert [r.sequence for r in spool.pending_since_watermark()] == [1, 2, 3, 4, 5] + + spool.advance_watermark(3) + assert spool.watermark() == 3 + assert [r.sequence for r in spool.pending_since_watermark()] == [4, 5] + + +def test_spool_watermark_rejects_a_regression_as_untrusted_input(tmp_path: Path) -> None: + # v1.3/M7: acked_through_sequence is untrusted platform input -- a value below the current + # watermark is rejected (typed error), not silently ignored; the watermark stays unchanged. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(0) + assert excinfo.value.code == "outbound_spool_watermark_out_of_range" + assert spool.watermark() == 1 + + +def test_spool_watermark_rejects_a_value_at_or_above_next_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) # next_sequence becomes 2 + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(2) # not yet allocated + assert excinfo.value.code == "outbound_spool_watermark_out_of_range" + assert spool.watermark() == 0 + + +def test_spool_watermark_advancing_to_the_same_value_is_a_no_op(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + spool.advance_watermark(1) # repeating the same ack must not raise + assert spool.watermark() == 1 + + +def test_spool_watermark_is_durable_across_a_restart(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + first.append({"event_id": "e0"}) + first.advance_watermark(1) + + OutboundSpool._forget_for_tests(tmp_path, "events") + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second.watermark() == 1 + + +def test_spool_watermark_degrades_to_zero_on_a_corrupt_file_instead_of_wedging(tmp_path: Path) -> None: + # M1: a corrupt/torn watermark file must not permanently brick the spool -- re-sending + # already-acked events is safe (at-least-once + dedupe on event_id); raising forever is not. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + watermark_path = tmp_path / "events.spool.watermark.json" + assert watermark_path.exists() + watermark_path.write_text("not json") + assert spool.watermark() == 0 + # And it self-heals: a fresh valid advance still works normally afterward. + spool.advance_watermark(1) + assert spool.watermark() == 1 + + +def test_spool_advance_watermark_leaves_no_leftover_temp_files(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.advance_watermark(1) + assert list(tmp_path.glob("*.tmp.*")) == [] + + +def test_advance_watermark_raises_on_a_forked_instance(tmp_path: Path) -> None: + # P7: `advance_watermark` is the one operation that DURABLY destroys delivery state -- once a + # value is written, `pending_since_watermark()` can never return anything at or below it again + # -- yet it was the one mutating method the N25 fork guard skipped. A forked child advancing the + # parent's watermark would silently orphan every record below the new value, the N1 outcome by + # a different route. `_poison_after_fork` is what `os.register_at_fork`'s `after_in_child` hook + # actually calls; invoking it directly simulates the fork without an actual `os.fork()`, which a + # shared test process can't safely drive. + from fi.alk.harness.outbound import _poison_after_fork + + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + _poison_after_fork() + + with pytest.raises(OutboundSpoolError) as excinfo: + spool.advance_watermark(1) + assert excinfo.value.code == "outbound_spool_forked" + + +def test_spool_recovery_seeds_next_sequence_from_the_watermark_when_the_log_is_lost( + tmp_path: Path, +) -> None: + """M5: a watermark ahead of the log (an M2-style lost file, or a compacted/dropped log) must + seed next_sequence from the watermark, not just the empty/short log -- otherwise the allocator + reissues sequence numbers the platform already closed.""" + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(7): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(7) + (tmp_path / "events.spool.jsonl").write_bytes(b"") # simulate the log vanishing + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 8 + assert recovered.append({"event_id": "e7"}).sequence == 8 + + +def test_spool_recovery_seeds_next_sequence_from_the_watermark_when_the_file_is_missing_entirely( + tmp_path: Path, +) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(4): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + (tmp_path / "events.spool.jsonl").unlink() + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 5 + + +def test_spool_drop_is_pure_physical_removal_and_never_touches_the_watermark( + tmp_path: Path, +) -> None: + # N1: an earlier version had `drop` also advance the watermark to `sequence` -- that let an + # UNTRUSTED `rejected[].sequence` silently orphan every pending record below it, bypassing + # `advance_watermark`'s own M7 clamp entirely. `drop`/`drop_many` are now pure physical removal; + # the batch-level `advance_watermark(acked_through_sequence)` is the only chokepoint. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) # sequences 1..5 + spool.drop(3) + remaining = [r.decode()["event_id"] for r in spool.records()] + assert remaining == ["e0", "e1", "e3", "e4"] # e2 was assigned sequence 3 + assert spool.watermark() == 0 # drop() must NEVER move the watermark + + +def test_spool_drop_many_batches_every_removal_into_one_rewrite(tmp_path: Path) -> None: + # N14: a batch of rejections is one _rewrite_retaining call, not one per sequence -- exercised + # indirectly here by checking the end result of a single drop_many call over a set. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(6): + spool.append({"event_id": f"e{i}"}) # sequences 1..6 + spool.drop_many({2, 4, 6}) + remaining = [r.decode()["event_id"] for r in spool.records()] + assert remaining == ["e0", "e2", "e4"] + assert spool.watermark() == 0 # still untouched -- drop_many never advances it either + + +def test_spool_drop_many_with_an_empty_set_is_a_no_op(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e0"}) + spool.drop_many(set()) + assert [r.decode()["event_id"] for r in spool.records()] == ["e0"] + + +def test_spool_compact_through_drops_acked_records_and_keeps_pending_ones(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(6): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + spool.compact_through(4) + assert [r.decode()["event_id"] for r in spool.records()] == ["e4", "e5"] + + # The allocator survives a restart correctly even though history was compacted away. + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.next_sequence == 7 + assert recovered.append({"event_id": "e6"}).sequence == 7 + + +def test_spool_compact_through_clamps_to_the_watermark_never_drops_pending_records( + tmp_path: Path, +) -> None: + # Fail-safe: a caller passing a too-high sequence must not be able to compact away undelivered + # records -- compaction is clamped to whatever the platform has actually acknowledged. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(5): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(2) + spool.compact_through(10**9) + assert [r.decode()["event_id"] for r in spool.records()] == ["e2", "e3", "e4"] + + +def test_spool_pending_since_watermark_uses_the_cached_offset_to_seek(tmp_path: Path) -> None: + # P11: previously this only asserted the RESULT ([5..10]), which the generic records_after() + # fallback produces identically -- it could not fail if the seek path regressed to the + # fallback. Monkeypatching records_after to raise pins the claim: pending_since_watermark() + # must succeed WITHOUT ever calling it, proving the cached-offset seek path was actually taken. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(10): + spool.append({"event_id": f"e{i}"}) + spool.advance_watermark(4) + + def _must_not_be_called(sequence: int) -> Any: + raise AssertionError("pending_since_watermark fell back to records_after instead of seeking") + + spool.records_after = _must_not_be_called # type: ignore[method-assign] + pending = spool.pending_since_watermark() + assert [r.sequence for r in pending] == list(range(5, 11)) + + +def test_unsequenced_spool_assigns_no_sequence(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + record = spool.append({"scenario_key": "k1", "status": "passed"}) + assert record.sequence is None + assert record.decode() == {"scenario_key": "k1", "status": "passed"} + + +def test_unsequenced_spool_rejects_a_caller_supplied_sequence_key(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError) as excinfo: + spool.append({"sequence": 1, "scenario_key": "k1"}) + # MIN-9: distinct from the accessor-misuse code below -- "caller passed a sequence key into an + # unsequenced append" and "caller called a sequence-only method" are different faults. + assert excinfo.value.code == "outbound_spool_caller_supplied_sequence" + + +@pytest.mark.parametrize( + "member", ["next_sequence", "watermark", "pending_since_watermark", "records_after", "drop", "compact_through"] +) +def test_unsequenced_spool_rejects_sequence_only_operations(tmp_path: Path, member: str) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError) as excinfo: + if member == "next_sequence": + spool.next_sequence + elif member in ("records_after", "drop", "compact_through"): + getattr(spool, member)(1) + else: + getattr(spool, member)() + assert excinfo.value.code == "outbound_spool_unsequenced" + + +def test_unsequenced_spool_rejects_advance_watermark(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "results", sequenced=False) + with pytest.raises(OutboundSpoolError): + spool.advance_watermark(1) + + +def test_separate_streams_do_not_share_sequence_numbers(tmp_path: Path) -> None: + events = OutboundSpool(tmp_path, "events", sequenced=True) + manifests = OutboundSpool(tmp_path, "artifact_manifest", sequenced=False) + events.append({"event_id": "e1"}) + events.append({"event_id": "e2"}) + manifests.append({"complete": False}) + assert events.next_sequence == 3 + assert [r.sequence for r in events.records()] == [1, 2] + assert [r.sequence for r in manifests.records()] == [None] + + +def test_spool_append_is_safe_under_concurrent_callers(tmp_path: Path) -> None: + # "one allocator, one lock" -- concurrent appends from multiple threads must still produce a + # contiguous, gap-free, duplicate-free sequence. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + sequences: list[int] = [] + lock = threading.Lock() + + def worker(start: int) -> None: + for i in range(25): + record = spool.append({"event_id": f"t{start}-{i}"}) + with lock: + sequences.append(record.sequence) + + threads = [threading.Thread(target=worker, args=(t,)) for t in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert sorted(sequences) == list(range(1, 201)) + assert spool.next_sequence == 201 + + +def test_spool_creates_its_root_directory(tmp_path: Path) -> None: + root = tmp_path / "nested" / "outbound" + assert not root.exists() + OutboundSpool(root, "events", sequenced=True) + assert root.is_dir() + + +def test_spool_root_directory_is_created_with_restrictive_permissions(tmp_path: Path) -> None: + # MIN-10: the spool holds event payloads and receipt bodies in a multi-user sandbox. + root = tmp_path / "outbound" + OutboundSpool(root, "events", sequenced=True) + assert (root.stat().st_mode & 0o777) == 0o700 + + +def test_spool_records_survive_being_read_back_after_many_appends(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(50): + spool.append({"event_id": f"e{i}", "n": i}) + records = spool.records() + assert len(records) == 50 + assert [r.decode()["n"] for r in records] == list(range(50)) + assert (tmp_path / "events.spool.jsonl").is_file() + + +# --- M6: one allocator per (root, name) ----------------------------------------------------- + + +def test_same_stream_name_returns_the_cached_instance_not_a_second_allocator(tmp_path: Path) -> None: + first = OutboundSpool(tmp_path, "events", sequenced=True) + second = OutboundSpool(tmp_path, "events", sequenced=True) + assert second is first + sequences = [first.append({"event_id": "a"}).sequence, second.append({"event_id": "b"}).sequence] + assert sequences == [1, 2] # one allocator regardless of how many handles point at it + + +def test_same_stream_name_with_a_conflicting_sequenced_flag_raises(tmp_path: Path) -> None: + OutboundSpool(tmp_path, "events", sequenced=True) + with pytest.raises(OutboundSpoolError) as excinfo: + OutboundSpool(tmp_path, "events", sequenced=False) + assert excinfo.value.code == "outbound_spool_sequenced_mismatch" + + +def test_flock_rejects_a_second_lock_holder_on_the_same_lock_file(tmp_path: Path) -> None: + # M6: cross-PROCESS protection -- the in-process registry above only protects against a second + # Python-level instance in this same interpreter; a genuinely separate process trying to open + # the same lock file must be rejected by the OS-level flock regardless of the registry. + OutboundSpool(tmp_path, "events", sequenced=True) # holds the flock via its own fd + lock_path = tmp_path / "events.spool.lock" + fd = os.open(str(lock_path), os.O_RDWR) + try: + with pytest.raises(OSError): + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + os.close(fd) + + +def test_a_forgotten_spool_can_be_reacquired_after_the_lock_is_released(tmp_path: Path) -> None: + OutboundSpool(tmp_path, "events", sequenced=True) + OutboundSpool._forget_for_tests(tmp_path, "events") # releases the flock fd too + # A fresh construction must succeed cleanly -- proves _forget_for_tests actually released the + # OS-level lock, not just the in-process cache entry. + fresh = OutboundSpool(tmp_path, "events", sequenced=True) + assert fresh.next_sequence == 1 + + +def test_close_releases_the_lock_and_evicts_the_registry(tmp_path: Path) -> None: + # N26: OutboundSpool had no supported teardown -- close() is the real one (_forget_for_tests is + # test-only and reaches for it internally now). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.close() + fresh = OutboundSpool(tmp_path, "events", sequenced=True) # must not raise outbound_spool_locked + assert fresh is not spool + assert fresh.next_sequence == 1 + spool.close() # idempotent -- a second close on the old instance must not raise + + +def test_close_poisons_the_instance_so_a_closed_spool_cannot_append_with_no_lock_held( + tmp_path: Path, +) -> None: + # P2: close() evicted the registry entry and released the flock fd, but left the closed + # instance itself fully mutable -- `_initialized` stayed True and nothing else guarded it, so a + # caller still holding the closed reference could keep appending with NO lock held, and a fresh + # construction for the same key would allocate an independent, unaware second `_next_sequence`. + # `close()` must poison the instance too, not just the registry slot. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + spool.append({"event_id": "e1"}) + spool.close() + + with pytest.raises(OutboundSpoolError) as excinfo: + spool.append({"event_id": "after-close"}) + assert excinfo.value.code == "outbound_spool_closed" + + with pytest.raises(OutboundSpoolError): + spool.advance_watermark(1) + + # A fresh instance for the same key is the ONLY live allocator now -- it continues the SAME + # sequence the closed instance left off at, rather than starting a second, independent one. + fresh = OutboundSpool(tmp_path, "events", sequenced=True) + fresh_record = fresh.append({"event_id": "fresh"}) + assert fresh_record.sequence == 2 + assert [r.decode()["event_id"] for r in fresh.records()] == ["e1", "fresh"] + + +def test_failed_construction_does_not_poison_the_registry_for_the_next_attempt(tmp_path: Path) -> None: + # N4, ranked missing test 4: a construction that fails PARTWAY (after acquiring the process + # lock, before finishing) must not leave a half-built instance registered -- the next + # construction attempt for the SAME key gets a typed error reflecting its OWN failure, never + # AttributeError (a half-built instance missing _sequenced) or a spurious outbound_spool_locked + # (a leaked fd from the first attempt). + root = tmp_path / "n4" + lock_path = root / "events.spool.lock" + root.mkdir() + # Simulate "another process" already holding the flock -- this is exactly what + # _acquire_process_lock hits AFTER self._sequenced is already set, so it reproduces the review's + # second failure mode (a failure after the half-built-instance point, not before it). + held_fd = os.open(str(lock_path), os.O_CREAT | os.O_RDWR, 0o600) + fcntl.flock(held_fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + with pytest.raises(OutboundSpoolError) as excinfo: + OutboundSpool(root, "events", sequenced=True) + assert excinfo.value.code == "outbound_spool_locked" + finally: + os.close(held_fd) # release the "other process"'s hold + + # The registry must have NOTHING registered for this key -- construction now succeeds cleanly, + # not with AttributeError ('_sequenced') and not with a second outbound_spool_locked from a + # leaked fd. + spool = OutboundSpool(root, "events", sequenced=True) + assert spool.next_sequence == 1 + + +# ================================================================================================= +# PART 2 -- Transport: error map, retry/backoff, typed receipt/manifest models, channel clients. +# ================================================================================================= + + +# --- classify_response (the closed error map) --------------------------------------------------- + + +def test_classify_response_returns_none_for_every_2xx() -> None: + assert classify_response(200, {}, attempt=1) is None + assert classify_response(201, {"status": "stored"}, attempt=1) is None + assert classify_response(299, {}, attempt=1) is None + + +def test_classify_response_network_failure_is_retryable_connectivity() -> None: + error = classify_response(None, None, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.domain is FailureDomain.CONNECTIVITY + assert error.code == "network_error" + + +@pytest.mark.parametrize("status", [401, 403]) +def test_classify_response_401_403_are_fenced_never_retried(status: int) -> None: + error = classify_response(status, {"error": "e", "message": "m", "retryable": False}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.FENCED + assert error.domain is None # "never an infra retry" -- not a FailureDomain-tagged outcome + + +def test_classify_response_404_retries_twice_then_channel_failed_on_the_third() -> None: + first = classify_response(404, {"error": "not_found", "message": "m"}, attempt=1) + second = classify_response(404, {"error": "not_found", "message": "m"}, attempt=2) + third = classify_response(404, {"error": "not_found", "message": "m"}, attempt=3) + assert first is not None and first.outcome is ChannelOutcome.RETRYABLE + assert second is not None and second.outcome is ChannelOutcome.RETRYABLE + assert third is not None + assert third.outcome is ChannelOutcome.CHANNEL_FAILED + assert third.domain is FailureDomain.PLATFORM_SYNC + + +def test_classify_response_413_is_budget_exceeded() -> None: + error = classify_response(413, {"error": "artifact_budget_exceeded", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert error.status_code == 413 + + +def test_classify_response_413_without_artifact_budget_exceeded_is_permanent_item() -> None: + # N7: 413 is Channel 3's code specifically -- a 413 that does NOT report + # artifact_budget_exceeded (e.g. an events batch too big for the platform's own ingress cap) + # must not be mislabeled BUDGET_EXCEEDED on a channel with no such concept. + error = classify_response(413, {"error": "request_entity_too_large", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + assert error.status_code == 413 + + error_no_body = classify_response(413, None, attempt=1) + assert error_no_body is not None + assert error_no_body.outcome is ChannelOutcome.PERMANENT_ITEM + + +def test_classify_response_404_domain_is_platform_sync_on_every_attempt() -> None: + # N29: a 404 is never a connectivity fault under §4.6 -- PLATFORM_SYNC on attempts 1-2 too, not + # just the third (cosmetic before; the inconsistency invited a wrong read). + first = classify_response(404, {"error": "not_found", "message": "m"}, attempt=1) + second = classify_response(404, {"error": "not_found", "message": "m"}, attempt=2) + assert first is not None and first.domain is FailureDomain.PLATFORM_SYNC + assert second is not None and second.domain is FailureDomain.PLATFORM_SYNC + + +def test_classify_response_unlisted_3xx_is_permanent_not_retryable() -> None: + # N27: the contract never contemplates a 3xx (every endpoint ends in "/" precisely so Django's + # POST-redirect problem never arises) -- a 301 from a mis-slashed endpoint should surface as a + # permanent misconfiguration, not loop max_attempts times before giving up as connectivity. + error = classify_response(301, {"error": "e", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + + +def test_classify_response_429_is_retryable_and_carries_retry_after() -> None: + error = classify_response( + 429, {"error": "rate_limited", "message": "m"}, attempt=1, retry_after_seconds=7.0 + ) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.retry_after_seconds == 7.0 + + +@pytest.mark.parametrize("status", [400, 409, 422]) +def test_classify_response_400_409_422_are_permanent_item(status: int) -> None: + error = classify_response(status, {"error": f"e{status}", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + assert error.domain is None + + +def test_classify_response_unlisted_4xx_is_the_permanent_catch_all() -> None: + error = classify_response(418, {"error": "teapot", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.PERMANENT_ITEM + + +@pytest.mark.parametrize("status", [500, 502, 503]) +def test_classify_response_5xx_is_retryable_connectivity(status: int) -> None: + error = classify_response(status, {"error": "e", "message": "m"}, attempt=1) + assert error is not None + assert error.outcome is ChannelOutcome.RETRYABLE + assert error.domain is FailureDomain.CONNECTIVITY + + +# --- compute_backoff_seconds (full jitter) -------------------------------------------------------- + + +def test_compute_backoff_seconds_full_jitter_formula() -> None: + # rng pinned to 1.0 -> exercises the pure ceiling formula, unjittered. + assert compute_backoff_seconds(1, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 1.0 + assert compute_backoff_seconds(2, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 2.0 + assert compute_backoff_seconds(3, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 4.0 + + +def test_compute_backoff_seconds_caps_at_max_backoff() -> None: + assert compute_backoff_seconds(10, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 1.0) == 15.0 + + +def test_compute_backoff_seconds_scales_by_rng() -> None: + assert compute_backoff_seconds(3, initial_backoff_seconds=1.0, max_backoff_seconds=15.0, rng=lambda: 0.5) == 2.0 + + +# --- format_rfc3339_millis ------------------------------------------------------------------------- + + +def test_format_rfc3339_millis_matches_the_contracts_wire_form() -> None: + assert format_rfc3339_millis(NOW) == "2026-08-25T10:14:03.412Z" + assert is_valid_rfc3339_millis(format_rfc3339_millis(NOW)) + + +def test_format_rfc3339_millis_rejects_naive_datetimes() -> None: + with pytest.raises(ValueError): + format_rfc3339_millis(datetime(2026, 8, 25)) + + +def test_is_valid_rfc3339_millis_rejects_the_default_pydantic_offset_form() -> None: + # The exact failure mode `format_rfc3339_millis` exists to prevent: pydantic's default + # datetime JSON serialization uses a `+00:00` offset and six-digit microseconds, not `Z` and + # milliseconds -- this string is what NOT using the helper would have produced. + assert not is_valid_rfc3339_millis("2026-08-25T10:14:03.412000+00:00") + + +# --- ArtifactBudgetTracker ------------------------------------------------------------------------- + + +def test_artifact_budget_tracker_reserved_kinds_always_admitted() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=10) + assert tracker.would_admit(ArtifactKind.RESULT, 1_000_000, digest="d1") + tracker.record(ArtifactKind.RESULT, 1_000_000, digest="d1") + # Reserved kinds are counted (for accounting) but never refused, even once over budget. + assert tracker.admitted_bytes == 1_000_000 + assert tracker.would_admit(ArtifactKind.BUILD, 999, digest="d2") + + +def test_artifact_budget_tracker_refuses_non_reserved_once_budget_is_exhausted() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=100) + assert tracker.would_admit(ArtifactKind.TRACE, 60, digest="d1") + tracker.record(ArtifactKind.TRACE, 60, digest="d1") + assert tracker.would_admit(ArtifactKind.TRACE, 40, digest="d2") + tracker.record(ArtifactKind.TRACE, 40, digest="d2") + assert not tracker.would_admit(ArtifactKind.LOG, 1, digest="d3") + + +def test_artifact_budget_tracker_duplicate_digest_is_free() -> None: + tracker = ArtifactBudgetTracker(max_artifact_bytes=10) + tracker.record(ArtifactKind.LOG, 10, digest="dup") + assert tracker.admitted_bytes == 10 + assert tracker.would_admit(ArtifactKind.LOG, 10, digest="dup") + tracker.record(ArtifactKind.LOG, 10, digest="dup") + assert tracker.admitted_bytes == 10 # unchanged -- already counted + + +def test_priority_class_orders_reserved_recordings_other() -> None: + # N16: reserved (0) > recordings (1) > trace/log/other (2), per the contract's three-tier + # budget partition. + assert priority_class(ArtifactKind.RESULT) == 0 + assert priority_class(ArtifactKind.BUILD) == 0 + assert priority_class(ArtifactKind.TRANSCRIPT) == 0 + assert priority_class(ArtifactKind.TOOL_TRACE) == 0 + assert priority_class(ArtifactKind.RECORDING_COMBINED) == 1 + assert priority_class(ArtifactKind.RECORDING_STEREO) == 1 + assert priority_class(ArtifactKind.TRACE) == 2 + assert priority_class(ArtifactKind.LOG) == 2 + assert priority_class(ArtifactKind.OTHER) == 2 + + +def test_artifact_budget_tracker_reserves_recording_headroom_from_trace_log_other() -> None: + # N16: a non-zero recording_headroom_bytes shrinks what a trace/log/other candidate may + # consume, leaving room for recordings not yet seen -- default (0) behavior is unaffected + # (covered by the pre-existing tracker tests above). + tracker = ArtifactBudgetTracker(max_artifact_bytes=100, recording_headroom_bytes=30) + # 70 bytes remain after the 30-byte reservation -- a 71-byte trace is refused, a 70-byte one is not. + assert not tracker.would_admit(ArtifactKind.TRACE, 71, digest="d1") + assert tracker.would_admit(ArtifactKind.TRACE, 70, digest="d2") + # Recordings are NOT subject to the headroom reservation themselves -- only class-2 candidates. + assert tracker.would_admit(ArtifactKind.RECORDING_COMBINED, 100, digest="d3") + + +# --- ResultReceiptDraft / build_result_receipt / build_skipped_receipt ----------------------------- + + +def _passed_receipt(**overrides: Any) -> dict[str, Any]: + base = dict( + job_id="j1", + attempt_id="a1", + attempt_number=1, + scenario_key="suspended-account-blocked", + scenario_id="sid-1", + scenario_attempt=1, + world_index=2, + status=ScenarioStatus.PASSED, + sub_goals=[{"name": "n", "held": True, "reason": None, "judged": False}], + evaluations=[ + {"name": "customer_agent_task_completion", "kind": "metric", "score": 0.86, "reason": "r"}, + {"name": "checked_refund", "kind": "checkpoint", "passed": True, "reason": "r"}, + ], + call={ + "started_at": format_rfc3339_millis(NOW), + "ended_at": format_rfc3339_millis(LATER), + "duration_ms": 184211, + "turns": 5, + "transcript_artifact": "sha256:" + "a" * 64, + "recording_artifacts": ["sha256:" + "b" * 64], + }, + failure=None, + ) + base.update(overrides) + return build_result_receipt(**base) + + +def test_build_result_receipt_matches_the_contract_shaped_example() -> None: + receipt = _passed_receipt() + assert is_valid_digest(receipt["digest"]) + assert receipt["status"] == "passed" + assert receipt["digest"] == whole_object_digest({k: v for k, v in receipt.items() if k != "digest"}) + ResultReceiptDraft.model_validate(receipt) # round-trips through the model unchanged + + +def test_build_skipped_receipt_has_the_exact_contract_shape() -> None: + receipt = build_skipped_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k2", scenario_id="sid-2" + ) + assert receipt["scenario_attempt"] == 1 + assert receipt["world_index"] is None + assert receipt["sub_goals"] == [] + assert receipt["evaluations"] == [] + assert receipt["call"] is None + assert receipt["failure"] is None + assert receipt["status"] == "skipped" + + +def test_skipped_shape_is_enforced_even_outside_the_synthesizing_helper() -> None: + with pytest.raises(ValidationError): + # world_index must be null for a skipped receipt -- even via the general-purpose builder. + _passed_receipt(status=ScenarioStatus.SKIPPED, world_index=0, sub_goals=[], evaluations=[], call=None) + + +def test_errored_receipt_requires_a_failure() -> None: + with pytest.raises(ValidationError): + _passed_receipt(status=ScenarioStatus.ERRORED, call=None, failure=None) + + errored = _passed_receipt( + status=ScenarioStatus.ERRORED, + call=None, + failure={"domain": "agent", "stage": "running", "code": "agent_crashed", "message": "m"}, + ) + assert is_valid_digest(errored["digest"]) + + +def test_receipt_evaluations_discriminated_union_rejects_a_mixed_up_kind() -> None: + with pytest.raises(ValidationError): + _passed_receipt(evaluations=[{"name": "n", "kind": "metric", "passed": True, "reason": "r"}]) + + +def test_receipt_call_timestamp_must_be_the_canonical_rfc3339_millis_form() -> None: + with pytest.raises(ValidationError): + _passed_receipt(call={**_passed_receipt()["call"], "started_at": "2026-08-25T10:14:03.412000+00:00"}) + + +def test_a_tampered_receipt_digest_is_rejected() -> None: + receipt = _passed_receipt() + receipt["digest"] = "sha256:" + "0" * 64 + with pytest.raises(ValidationError): + ResultReceiptDraft.model_validate(receipt) + + +def test_receipt_digest_mismatch_names_the_unset_default_field() -> None: + # N23: a caller whose raw `call` dict omits `recording_artifacts` (an optional field with a + # default) computes a digest over an object without it; the model's own re-derivation fills in + # `recording_artifacts: []`, so the two digests disagree. The error should name the differing + # field -- a bare `receipt_digest_mismatch` was the review's named "opaque, undiagnosable" gap. + call_without_recording_artifacts = { + "started_at": format_rfc3339_millis(NOW), + "ended_at": format_rfc3339_millis(LATER), + "duration_ms": 100, + "turns": 1, + "transcript_artifact": None, + # "recording_artifacts" intentionally omitted -- CallSummary fills it via default_factory. + } + with pytest.raises(ValidationError) as excinfo: + build_result_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid", + scenario_attempt=1, world_index=0, status=ScenarioStatus.PASSED, + sub_goals=[{"name": "n", "held": True, "reason": None, "judged": False}], + evaluations=[], + call=call_without_recording_artifacts, + failure=None, + ) + assert "call.recording_artifacts" in str(excinfo.value) + + +# --- ArtifactManifestDraft / build_artifact_manifest ------------------------------------------------ + + +def test_build_artifact_manifest_matches_the_contract_shaped_example() -> None: + manifest = build_artifact_manifest( + job_id="j1", + attempt_id="a1", + attempt_number=1, + entries=[{"artifact_id": "sha256:" + "1" * 64, "kind": "result", "size": 1048576, "scenario_key": "k"}], + complete=True, + ) + assert is_valid_digest(manifest["digest"]) + ArtifactManifestDraft.model_validate(manifest) + + +def test_manifest_complete_flag_changes_the_digest() -> None: + entries = [{"artifact_id": "sha256:" + "2" * 64, "kind": "log", "size": 10, "scenario_key": None}] + complete_manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=entries, complete=True) + partial_manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=entries, complete=False) + # "a later `complete: true` manifest is a distinct document that supersedes an earlier + # `complete: false` one from the same attempt." + assert complete_manifest["digest"] != partial_manifest["digest"] + + +def test_manifest_entry_rejects_a_malformed_artifact_id() -> None: + with pytest.raises(ValidationError): + build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": "not-a-digest", "kind": "log", "size": 1, "scenario_key": None}], + complete=True, + ) + + +# --- FakePlatform: in-process, no-sockets Transport ------------------------------------------------ + + +class FakePlatform: + """A `Transport` implementation backed by real, persistent server-side state -- no sockets, no + `requests` -- so the channel clients above are exercised through the exact same interface + `RequestsTransport` implements. `programmed` lets a test inject canned failures (an `Exception` + instance to raise, or a `TransportResponse` to return) that are consumed FIFO before requests + fall through to the real per-channel handlers below, which maintain idempotent state matching + the contract's own dedupe rules (`event_id`, `(job_id, scenario_key)`, artifact digest, + `(attempt_id, digest)`). `crash_after_next_write` lets a test simulate "the platform durably + processed the write, but the response never reached the guest" -- the handler still records + state and THEN raises `TransportError`, so the next request (whether inside the same client + call's retry loop, or from a brand new client after a simulated restart) hits the idempotent + path rather than reprocessing. + """ + + def __init__( + self, + *, + events_url: str, + results_url: str, + artifacts_url: str, + expected_token: str = "tok", + expected_fence: str = "fence1", + ) -> None: + self.events_url = events_url + self.results_url = results_url + self.artifacts_url = artifacts_url + self.manifest_url = artifacts_url + "manifest/" + self.expected_token = expected_token + self.expected_fence = expected_fence + + self._events_by_id: dict[str, dict[str, Any]] = {} + self._watermark = 0 + self._receipts: dict[tuple[str, str], dict[str, Any]] = {} + self._artifacts: dict[str, bytes] = {} + self._manifests: dict[tuple[str, str], dict[str, Any]] = {} + + self.events_to_reject: set[str] = set() + self.budget_remaining: int | None = None + self.reserved_kinds = {"build", "transcript", "tool_trace", "result"} + + self.programmed: list[Exception | TransportResponse] = [] + self.crash_after_next_write = False + self.calls: list[tuple[str, str]] = [] + self.received_headers: list[dict[str, str]] = [] + + def queue(self, item: Exception | TransportResponse) -> None: + self.programmed.append(item) + + def _maybe_crash(self, response: TransportResponse) -> TransportResponse: + if self.crash_after_next_write: + self.crash_after_next_write = False + raise TransportError("simulated crash: response lost after platform-side processing") + return response + + def _check_auth(self, headers: dict[str, str]) -> TransportResponse | None: + # Strengthened per the union review: every real (non-programmed) request must carry the + # bearer + fence -- exercising the auth-header wiring the contract mandates on every one of + # the four endpoints, not just implicitly via "the call succeeded." + if headers.get("Authorization") != f"Bearer {self.expected_token}": + return TransportResponse(401, {"error": "token_invalid", "message": "missing/wrong bearer", "retryable": False}, {}) + if headers.get("X-Harness-Fence") != self.expected_fence: + return TransportResponse(403, {"error": "fence_mismatch", "message": "missing/wrong fence", "retryable": False}, {}) + return None + + def request( + self, + method: str, + url: str, + *, + headers: dict[str, str], + json_body: dict[str, Any] | None = None, + data: Any = None, + timeout: float = 30.0, + ) -> TransportResponse: + self.calls.append((method, url)) + self.received_headers.append(dict(headers)) + if self.programmed: + item = self.programmed.pop(0) + if isinstance(item, Exception): + raise item + return item + + auth_error = self._check_auth(headers) + if auth_error is not None: + return auth_error + + if method == "POST" and url == self.events_url: + # EventsClient.flush() sends the batch as spooled bytes verbatim via `data=` (N19), not + # `json_body=` -- decode it the same way the real platform would. + if json_body is not None: + parsed_body = json_body + elif isinstance(data, (bytes, bytearray)): + parsed_body = json.loads(data.decode("utf-8")) + else: + parsed_body = {} + return self._handle_events(parsed_body) + if method == "POST" and url == self.results_url: + return self._handle_receipt(json_body or {}) + if method == "POST" and url == self.manifest_url: + return self._handle_manifest(json_body or {}) + if method == "PUT" and url.startswith(self.artifacts_url): + return self._handle_upload(url, headers, data) + raise AssertionError(f"unrouted fake-platform request: {method} {url}") + + def _handle_events(self, body: dict[str, Any]) -> TransportResponse: + assert body.get("schema_version") == "futureagi.harness-event.v1", ( + f"batch envelope missing/wrong schema_version: {body.get('schema_version')!r}" + ) + rejected: list[dict[str, Any]] = [] + max_seq = self._watermark + for event in body["events"]: + sequence = event["sequence"] + max_seq = max(max_seq, sequence) + if event["event_id"] in self.events_to_reject: + rejected.append( + { + "event_id": event["event_id"], + "sequence": sequence, + "code": "deterministic_rejection", + "message": "rejected by fake", + } + ) + continue + self._events_by_id[event["event_id"]] = event + self._watermark = max(self._watermark, max_seq) + return self._maybe_crash( + TransportResponse(200, {"acked_through_sequence": self._watermark, "rejected": rejected}, {}) + ) + + def _handle_receipt(self, body: dict[str, Any]) -> TransportResponse: + # Strengthened per the union review: the real platform verifies the receipt digest -- + # recompute it the same way `whole_object_digest` does and reject a mismatch, so a test that + # tampers with a receipt body actually exercises this path instead of it being silently + # unenforced. + core = {k: v for k, v in body.items() if k != "digest"} + if body.get("digest") != whole_object_digest(core): + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + key = (body["job_id"], body["scenario_key"]) + existing = self._receipts.get(key) + if existing is not None: + if existing["digest"] == body["digest"]: + return self._maybe_crash(TransportResponse(200, {"status": "duplicate"}, {})) + if existing["attempt_number"] >= body["attempt_number"]: + code = ( + "attempt_superseded" + if existing["attempt_number"] > body["attempt_number"] + else "receipt_conflict" + ) + return self._maybe_crash( + TransportResponse(409, {"error": code, "message": "m", "retryable": False}, {}) + ) + self._receipts[key] = body + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + def _handle_upload(self, url: str, headers: dict[str, str], data: Any) -> TransportResponse: + digest_hex = url[len(self.artifacts_url) :].rstrip("/") + body_bytes = data if isinstance(data, (bytes, bytearray)) else b"".join(data) + if digest_hex in self._artifacts: + return self._maybe_crash(TransportResponse(200, {"status": "already_exists"}, {})) + if hashlib.sha256(body_bytes).hexdigest() != digest_hex: + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + kind = headers.get("X-Artifact-Kind") + if ( + self.budget_remaining is not None + and kind not in self.reserved_kinds + and len(body_bytes) > self.budget_remaining + ): + return self._maybe_crash( + TransportResponse( + 413, {"error": "artifact_budget_exceeded", "message": "m", "retryable": False}, {} + ) + ) + self._artifacts[digest_hex] = body_bytes + if self.budget_remaining is not None: + self.budget_remaining -= len(body_bytes) + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + def _handle_manifest(self, body: dict[str, Any]) -> TransportResponse: + # Strengthened per the union review: recompute and verify the manifest digest too. + core = {k: v for k, v in body.items() if k != "digest"} + if body.get("digest") != whole_object_digest(core): + return self._maybe_crash( + TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {}) + ) + key = (body["attempt_id"], body["digest"]) + if key in self._manifests: + return self._maybe_crash(TransportResponse(200, {"status": "duplicate"}, {})) + for entry in body["entries"]: + digest_hex = entry["artifact_id"].split(":", 1)[1] + if digest_hex not in self._artifacts: + return self._maybe_crash( + TransportResponse(422, {"error": "artifact_unknown", "message": entry["artifact_id"], "retryable": False}, {}) + ) + self._manifests[key] = body + return self._maybe_crash(TransportResponse(201, {"status": "stored"}, {})) + + +ENDPOINTS = { + "events": "https://platform.example/events/", + "results": "https://platform.example/results/", + "artifacts": "https://platform.example/artifacts/", + "scenarios": "https://platform.example/scenarios/", +} + + +def _capabilities() -> HostedCapabilities: + return HostedCapabilities.model_validate( + { + "schema_version": CAPABILITIES_SCHEMA_VERSION, + "job_id": "j1", + "attempt_id": "a1", + "attempt_number": 1, + "fence": "fence1", + "expires_at": "2099-01-01T00:00:00.000Z", + "token": "tok", + "endpoints": ENDPOINTS, + } + ) + + +def _platform() -> FakePlatform: + return FakePlatform( + events_url=ENDPOINTS["events"], results_url=ENDPOINTS["results"], artifacts_url=ENDPOINTS["artifacts"] + ) + + +def _spooled_log_event(spool: OutboundSpool, event_id: str) -> None: + record = build_event_record( + event_id=event_id, + job_id="j1", + attempt_id="a1", + attempt_number=1, + emitted_at=NOW, + stage=HarnessStage.RUNNING, + type=OutboundEventType.LOG, + payload={"level": "info", "message": event_id}, + ) + spool.append(record) + + +@pytest.mark.parametrize("channel", ["events", "results", "artifacts_upload", "artifacts_manifest"]) +def test_every_channel_client_sends_authorization_and_fence_headers(tmp_path: Path, channel: str) -> None: + # Ranked missing test 8: one parametrized test over all four endpoints. `FakePlatform` now + # enforces these headers itself (`_check_auth`) -- a wrong/missing one would 401/403 here, so a + # passing run proves the header is both sent AND correct, not merely present in a recorded list. + platform = _platform() + capabilities = _capabilities() + if channel == "events": + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + result = EventsClient(capabilities, spool, platform, sleep=lambda s: None).flush() + assert result.error is None + elif channel == "results": + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + result = ResultsClient(capabilities, platform, sleep=lambda s: None).push(receipt) + assert result.delivered + elif channel == "artifacts_upload": + data = b"auth header check" + digest_hex = hashlib.sha256(data).hexdigest() + result = ArtifactsClient(capabilities, platform, sleep=lambda s: None).upload(digest_hex, data, kind=ArtifactKind.LOG) + assert result.delivered + else: + manifest = build_artifact_manifest(job_id="j1", attempt_id="a1", attempt_number=1, entries=[], complete=True) + result = ArtifactsClient(capabilities, platform, sleep=lambda s: None).push_manifest(manifest) + assert result.delivered + + assert platform.received_headers # at least one request actually happened + for headers in platform.received_headers: + assert headers.get("Authorization") == "Bearer tok" + assert headers.get("X-Harness-Fence") == "fence1" + + +# --- EventsClient ------------------------------------------------------------------------------ + + +def test_events_client_flush_delivers_and_advances_the_watermark(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(3): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + assert spool.watermark() == 0 + result = client.flush() + assert result.error is None + assert result.delivered_count == 3 + assert result.acked_through_sequence == 3 + assert result.rejected == [] + assert spool.watermark() == 3 + + +def test_events_client_flush_is_a_no_op_transport_call_when_nothing_is_pending(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + result = client.flush() + assert result.delivered_count == 0 + assert platform.calls == [] + + +def test_events_client_never_advances_watermark_on_a_retryable_failure(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportError("boom")) + client = EventsClient( + _capabilities(), spool, platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=1) + ) + result = client.flush() + assert result.error is not None + assert result.error.outcome is ChannelOutcome.RETRYABLE + assert spool.watermark() == 0 # "advancing the spool watermark only on confirmed delivery" + + +def test_events_client_transient_then_success_network_error_then_5xx(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportError("connection refused")) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert result.delivered_count == 1 + assert sleeps == [1.0, 2.0] # base backoff, then doubled -- full jitter pinned to 1.0 + assert len(platform.calls) == 3 + + +def test_events_client_429_honors_retry_after_over_computed_backoff(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "9"})) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [9.0] + + +def test_events_client_retry_after_86400_is_clamped_to_max_backoff(tmp_path: Path) -> None: + # N5, ranked missing test 6: an absurd Retry-After must not be honored verbatim -- clamp to + # retry_policy.max_backoff_seconds (15.0 by default). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue( + TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "86400"}) + ) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [15.0] + + +def test_events_client_retry_after_negative_is_treated_as_absent(tmp_path: Path) -> None: + # N5: a negative Retry-After must not reach `time.sleep` (ValueError) -- treated as absent, the + # caller falls back to the computed full-jitter backoff instead. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue( + TransportResponse(429, {"error": "rate_limited", "message": "m", "retryable": True}, {"Retry-After": "-5"}) + ) + sleeps: list[float] = [] + client = EventsClient(_capabilities(), spool, platform, sleep=sleeps.append, rng=lambda: 1.0) + + result = client.flush() + assert result.error is None + assert sleeps == [1.0] # the computed backoff for attempt 1, not -5.0 and not a crash + + +def test_perform_with_retry_deadline_refuses_a_new_attempt_once_elapsed() -> None: + from fi.alk.harness.outbound import RetryPolicy, _perform_with_retry + + calls: list[int] = [] + + def perform(attempt: int) -> TransportResponse: + calls.append(attempt) + return TransportResponse(503, {"error": "e", "message": "m", "retryable": True}, {}) + + response, error = _perform_with_retry( + perform, + retry_policy=RetryPolicy(max_attempts=8), + sleep=lambda s: None, + now=lambda: 100.0, + deadline=99.0, # already in the past relative to `now` + ) + assert calls == [] # never even attempted once + assert response is None + assert error is not None + assert error.code == "deadline_exceeded" + + +def test_perform_with_retry_deadline_clamps_the_sleep_to_the_remaining_budget() -> None: + from fi.alk.harness.outbound import RetryPolicy, _perform_with_retry + + clock = [0.0] + + def now() -> float: + return clock[0] + + def perform(_attempt: int) -> TransportResponse: + return TransportResponse(503, {"error": "e", "message": "m", "retryable": True}, {}) + + sleeps: list[float] = [] + + def sleep(seconds: float) -> None: + sleeps.append(seconds) + clock[0] += seconds + + _perform_with_retry( + perform, + retry_policy=RetryPolicy(max_attempts=8, initial_backoff_seconds=100.0, max_backoff_seconds=100.0), + sleep=sleep, + rng=lambda: 1.0, + now=now, + deadline=5.0, # far less than the ~100s computed backoff + ) + assert sleeps[0] == 5.0 # clamped to what remained, not the full computed backoff + + +def test_events_client_deterministic_rejection_advances_watermark_but_never_retries(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_ok") + _spooled_log_event(spool, "event_bad") + platform = _platform() + platform.events_to_reject = {"event_bad"} + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == 2 + assert [item["event_id"] for item in result.rejected] == ["event_bad"] + assert result.delivered_count == 1 + assert spool.watermark() == 2 # "the watermark is highest-processed -- accepted AND rejected" + assert len(platform.calls) == 1 # never retried + + +def test_events_client_physically_drops_a_rejected_record_from_the_spool(tmp_path: Path) -> None: + # P8 alignment: rejected-event handling must go through the spool's own drop(sequence), not + # leave the rejected record sitting in the log forever. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_ok") + _spooled_log_event(spool, "event_bad") + platform = _platform() + platform.events_to_reject = {"event_bad"} + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + client.flush() + remaining_ids = [record.decode()["event_id"] for record in spool.records()] + assert "event_bad" not in remaining_ids + assert "event_ok" in remaining_ids + + +def test_events_client_flush_rejection_does_not_destroy_an_unacked_record_past_a_corrupt_spool( + tmp_path: Path, +) -> None: + # P1 (BLOCKER->MAJOR), reproduces the round-3 review's exact end-to-end scenario through the + # public API: `_rewrite_retaining` (what the rejection-driven `drop_many` funnels through) used + # to rewrite the log from ONLY the records-before-corruption prefix -- so an ordinary, + # contract-defined rejection during `flush()` silently destroyed every intact record past the + # corrupt byte too, including one the platform had NOT even acknowledged yet (the terminal + # event, in the worst case). This is the N1 failure class (silent, zero-diagnostic loss of the + # outbound event stream) reintroduced by the N8/N14 rework's interaction, and it is the one + # thing round 3 was explicitly asked to break. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_1") + + path = tmp_path / "events.spool.jsonl" + lines = path.read_bytes().split(b"\n") + lines[0] = b"{not valid json but has a trailing newline" + path.write_bytes(b"\n".join(lines)) + + OutboundSpool._forget_for_tests(tmp_path, "events") + recovered = OutboundSpool(tmp_path, "events", sequenced=True) + assert recovered.is_corrupt + + _spooled_log_event(recovered, "event_bad") # sequence 1, appended post-recovery + _spooled_log_event(recovered, "event_terminal") # sequence 2, genuinely never yet acked + + platform = _platform() + # Ack ONLY sequence 1 (rejected) -- sequence 2 (event_terminal) is deliberately left un-acked, + # so it must still be reported pending after this flush, not silently vanish from disk. + platform.queue( + TransportResponse( + 200, + { + "acked_through_sequence": 1, + "rejected": [{"sequence": 1, "code": "event_type_unknown", "message": "m"}], + }, + {}, + ) + ) + client = EventsClient(_capabilities(), recovered, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + + pending_ids = [r.decode()["event_id"] for r in recovered.pending_since_watermark()] + assert pending_ids == ["event_terminal"], ( + "P1 regression: an un-acked record past a corrupt byte was destroyed by the rewrite the " + "rejection triggered" + ) + assert b"event_terminal" in path.read_bytes() + + +def test_events_client_rejects_an_untrusted_acked_through_sequence_and_leaves_watermark_unchanged( + tmp_path: Path, +) -> None: + # M7: acked_through_sequence is untrusted platform input -- a value outside the spool's trusted + # range must not be applied; the flush still reports success (the HTTP call itself delivered). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 10**9, "rejected": []}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.ack_out_of_range is True + assert result.error is None + assert spool.watermark() == 0 + + +def test_events_client_rejected_sequence_outside_the_batch_is_ignored_not_orphaning( + tmp_path: Path, +) -> None: + # N1 (BLOCKER), ranked missing test 1: a `rejected[].sequence` the guest never sent in this + # batch must be ignored -- not dropped, not used to move anything -- so it cannot orphan pending + # records. Reproduces the union review's exact scenario: 10 spooled events, watermark forced to + # 0, a rejected entry naming a sequence far outside [1, 10]. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(10): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + platform.queue( + TransportResponse(200, {"acked_through_sequence": 0, "rejected": [{"event_id": "x9", "sequence": 10**9, "code": "c", "message": "m"}]}, {}) + ) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.rejected == [] # the bogus entry never makes it into the reported rejections + assert spool.watermark() == 0 + assert len(spool.pending_since_watermark()) == 10 # nothing orphaned + assert len(spool.records()) == 10 # nothing physically dropped either + + +def test_events_client_ack_body_missing_acked_through_sequence_sets_ack_missing( + tmp_path: Path, +) -> None: + # N2/N3, ranked missing test 1: a 2xx with no acked_through_sequence at all must not silently + # re-send the same batch forever -- ack_missing is set, watermark stays put, no exception. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"rejected": []}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.ack_missing is True + assert spool.watermark() == 0 + + +@pytest.mark.parametrize( + "body", + [ + {"acked_through_sequence": None, "rejected": []}, + {"acked_through_sequence": "abc", "rejected": []}, + {"acked_through_sequence": True, "rejected": []}, # bool must not pass an int check + {}, + ], +) +def test_events_client_hostile_ack_bodies_never_raise(tmp_path: Path, body: dict[str, Any]) -> None: + # N2, ranked missing test 1: a non-int/missing acked_through_sequence must produce a typed, + # non-exception result, never crash the flusher (`TypeError`/`ValueError` from a bare `int()`). + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, body, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.ack_missing is True + assert spool.watermark() == 0 + + +def test_events_client_rejected_is_not_a_list_never_raises(tmp_path: Path) -> None: + # N2: `rejected: null` (or any non-list) must be treated as empty, not crash `sorted()`/iteration. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 1, "rejected": None}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.rejected == [] + assert spool.watermark() == 1 + + +def test_events_client_rejected_list_of_non_dicts_never_raises(tmp_path: Path) -> None: + # N2: `rejected: ["oops"]` (strings, not objects) must not crash `entry.get("sequence")`. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(200, {"acked_through_sequence": 1, "rejected": ["oops", 5, None]}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() # must not raise + assert result.error is None + assert result.rejected == [] + assert spool.watermark() == 1 + + +def test_events_client_503_503_404_does_not_raise_channel_failed(tmp_path: Path) -> None: + # N6: interleaved 5xx must not shorten the 404 budget -- only 3 OBSERVED 404s (not 3 attempts + # total) reach CHANNEL_FAILED. The ranked missing test names this sequence exactly. + # max_attempts=3 stops the retry loop right after this exact sequence (rather than letting it + # roll into a 4th, real-success attempt against FakePlatform, which would also prove the point + # but less precisely) -- classify_response must still read the single 404 as RETRYABLE, not + # CHANNEL_FAILED, because only ONE 404 has been observed. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient( + _capabilities(), spool, platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=3) + ) + + result = client.flush() # must NOT raise HostedChannelFailedError -- only one 404 seen so far + assert result.error is not None + assert result.error.outcome is ChannelOutcome.RETRYABLE + assert result.error.status_code == 404 + assert len(platform.calls) == 3 + + +def test_events_client_503_503_404_404_404_raises_channel_failed(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + platform.queue(TransportResponse(503, {"error": "service_unavailable", "message": "m", "retryable": True}, {})) + for _ in range(3): + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedChannelFailedError) as excinfo: + client.flush() + assert excinfo.value.error.domain is FailureDomain.PLATFORM_SYNC + assert len(platform.calls) == 5 + + +def test_events_client_413_on_the_events_channel_does_not_forever_loop(tmp_path: Path) -> None: + # N7, ranked missing test 7: a 413 with no artifact_budget_exceeded body classifies as a + # PERMANENT_ITEM (not a nonsensical BUDGET_EXCEEDED on a non-artifact channel) so the batch does + # not silently re-send forever with no diagnostic. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(413, {"error": "request_entity_too_large", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is not None + assert result.error.outcome is ChannelOutcome.PERMANENT_ITEM + assert result.error.status_code == 413 + + +def test_events_client_413_halves_the_batch_and_retries(tmp_path: Path) -> None: + # N7: a 413 that DOES report artifact_budget_exceeded (unexpected on this channel, but the + # reactive halving does not care why) triggers a halve-and-retry rather than giving up outright. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(4): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + platform.queue(TransportResponse(413, {"error": "artifact_budget_exceeded", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + # First attempt (batch of 4) 413s; second attempt (batch of 2) succeeds. + assert result.acked_through_sequence == 2 + assert len(spool.pending_since_watermark()) == 2 + + +def test_events_client_channel_state_latches_a_fence_and_stops_touching_the_transport( + tmp_path: Path, +) -> None: + # N10: once fenced, EVERY subsequent call on a client sharing this ChannelState raises the SAME + # error immediately, without ever hitting the transport again. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + _spooled_log_event(spool, "event_y") + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + state = ChannelState() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, channel_state=state) + + with pytest.raises(HostedFencedError): + client.flush() + calls_after_first = len(platform.calls) + + with pytest.raises(HostedFencedError) as excinfo: + client.flush() + assert len(platform.calls) == calls_after_first # no new transport call at all + assert excinfo.value.error.code == "token_expired" + + +def test_channel_state_shared_across_clients_fences_all_of_them(tmp_path: Path) -> None: + # N10: the fence is per-ATTEMPT, not per-channel -- a fence observed on EventsClient must also + # stop a ResultsClient sharing the same ChannelState, with no transport call from the second. + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(403, {"error": "fence_invalid", "message": "m", "retryable": False}, {})) + state = ChannelState() + events_client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, channel_state=state) + results_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, channel_state=state) + + with pytest.raises(HostedFencedError): + events_client.flush() + calls_before = len(platform.calls) + + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + with pytest.raises(HostedFencedError) as excinfo: + results_client.push(receipt) + assert len(platform.calls) == calls_before # results_client never touched the transport + assert excinfo.value.error.code == "fence_invalid" + + +def test_events_client_401_raises_hosted_fenced_error(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedFencedError) as excinfo: + client.flush() + assert excinfo.value.error.code == "token_expired" + assert spool.watermark() == 0 + + +def test_events_client_404_exhausted_three_times_raises_hosted_channel_failed_error(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_x") + platform = _platform() + for _ in range(3): + platform.queue(TransportResponse(404, {"error": "not_found", "message": "m", "retryable": False}, {})) + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + with pytest.raises(HostedChannelFailedError) as excinfo: + client.flush() + assert excinfo.value.error.domain is FailureDomain.PLATFORM_SYNC + assert len(platform.calls) == 3 + + +def test_events_client_crash_between_send_and_ack_redelivers_safely_within_one_flush(tmp_path: Path) -> None: + spool = OutboundSpool(tmp_path, "events", sequenced=True) + _spooled_log_event(spool, "event_crash") + platform = _platform() + platform.crash_after_next_write = True + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == 1 + assert spool.watermark() == 1 + assert len(platform.calls) == 2 + assert "event_crash" in platform._events_by_id + + +def test_events_client_batch_size_is_clamped_to_the_contract_max(tmp_path: Path) -> None: + from fi.alk.harness.outbound import EVENTS_MAX_BATCH + + spool = OutboundSpool(tmp_path, "events", sequenced=True) + for i in range(EVENTS_MAX_BATCH + 10): + _spooled_log_event(spool, f"event_{i}") + platform = _platform() + client = EventsClient(_capabilities(), spool, platform, sleep=lambda s: None, batch_size=10_000) + + result = client.flush() + assert result.error is None + assert result.acked_through_sequence == EVENTS_MAX_BATCH + assert spool.watermark() == EVENTS_MAX_BATCH + # Everything past the batch cap is still pending for the next flush. + assert len(spool.pending_since_watermark()) == 10 + + +# --- ResultsClient ------------------------------------------------------------------------------- + + +def test_results_client_pushes_a_receipt_successfully() -> None: + platform = _platform() + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + result = client.push(receipt) + assert result.delivered + assert not result.already_existed + assert platform._receipts[("j1", "k")]["digest"] == receipt["digest"] + + +def test_results_client_permanent_item_is_never_retried() -> None: + platform = _platform() + platform.queue(TransportResponse(422, {"error": "artifact_unknown", "message": "m", "retryable": False}, {})) + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + + result = client.push(receipt) + assert not result.delivered + assert result.error is not None + assert result.error.outcome is ChannelOutcome.PERMANENT_ITEM + assert result.error.code == "artifact_unknown" + assert len(platform.calls) == 1 + + +def test_results_client_401_raises_hosted_fenced_error() -> None: + platform = _platform() + platform.queue(TransportResponse(401, {"error": "token_expired", "message": "m", "retryable": False}, {})) + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + with pytest.raises(HostedFencedError): + client.push({"job_id": "j1", "scenario_key": "k", "digest": "sha256:" + "0" * 64, "attempt_number": 1}) + + +def test_results_client_crash_between_send_and_ack_redelivers_within_one_push() -> None: + platform = _platform() + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + platform.crash_after_next_write = True + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + + result = client.push(receipt) + assert result.delivered + assert result.already_existed # the retry loop's own second attempt saw the already-stored write + assert len(platform.calls) == 2 + + +def test_results_client_crash_between_send_and_ack_redelivers_across_a_simulated_restart() -> None: + platform = _platform() + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + platform.crash_after_next_write = True + # max_attempts=1: the retry loop can't self-heal, so the crash surfaces as a RETRYABLE return + # from this one push() call -- as if the guest process died right here. + first_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, retry_policy=RetryPolicy(max_attempts=1)) + first = first_client.push(receipt) + assert not first.delivered + assert first.error is not None and first.error.outcome is ChannelOutcome.RETRYABLE + + # A brand new client (simulating a fresh process) retries the same receipt: idempotent duplicate. + second_client = ResultsClient(_capabilities(), platform, sleep=lambda s: None) + second = second_client.push(receipt) + assert second.delivered + assert second.already_existed + assert len(platform.calls) == 2 + + +def test_results_client_attempt_superseded_latches_the_shared_channel_state() -> None: + # N22: 409 attempt_superseded is item-level PERMANENT_ITEM for the ONE call that received it + # (contract-correct -- still returned normally, not raised), but folds into the N10 latch for + # every SUBSEQUENT call: "a fenced attempt's in-flight requests cannot land after registration + # of its successor" is a fence in substance. + platform = _platform() + platform.queue(TransportResponse(409, {"error": "attempt_superseded", "message": "m", "retryable": False}, {})) + state = ChannelState() + client = ResultsClient(_capabilities(), platform, sleep=lambda s: None, channel_state=state) + receipt = build_skipped_receipt(job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid") + + first = client.push(receipt) + assert not first.delivered + assert first.error is not None and first.error.code == "attempt_superseded" + + with pytest.raises(HostedAttemptSupersededError): + client.push(receipt) + assert len(platform.calls) == 1 # the second call never touched the transport + + +def test_build_event_record_redacts_a_dsn_in_world_unhealthy_cause() -> None: + # N9, ranked missing test 9: outbound-channels.md v1.3's redaction obligation + seams v1.11 + # §3's "any outbound projection ... redacts userinfo" example DSN shape, verbatim. + record = _event( + OutboundEventType.WORLD_UNHEALTHY, + HarnessStage.RUNNING, + {"world_index": 2, "cause": "connect failed: postgresql://harness:s3cr3t@localhost:14000/w0"}, + ) + cause = record["payload"]["cause"] + assert "s3cr3t" not in cause + assert cause == "connect failed: postgresql://harness:***@localhost:14000/w0" + # The digest must match the REDACTED bytes actually spooled, not the original with the secret. + assert record["digest"] == event_payload_digest(record["payload"]) + + +def test_build_event_record_redacts_terminal_failure_message() -> None: + record = _event( + OutboundEventType.TERMINAL, + HarnessStage.FAILED, + { + "stage": "failed", + "reason": None, + "failure": { + "domain": "infrastructure", + "stage": "running", + "code": "world_pool_exhausted", + "message": "dial postgresql://harness:hunter2@localhost:14000/w0: connection refused", + }, + "scenario_counts": {"passed": 0, "failed": 0, "errored": 0, "skipped": 0}, + }, + ) + message = record["payload"]["failure"]["message"] + assert "hunter2" not in message + assert "postgresql://harness:***@localhost:14000/w0" in message + + +def test_build_result_receipt_redacts_sub_goal_and_evaluation_reasons_and_failure_message() -> None: + receipt = build_result_receipt( + job_id="j1", attempt_id="a1", attempt_number=1, scenario_key="k", scenario_id="sid", + scenario_attempt=1, world_index=None, + status=ScenarioStatus.ERRORED, + sub_goals=[{"name": "n", "held": None, "reason": "leaked postgresql://harness:pw123@host/db", "judged": False}], + evaluations=[{"name": "n2", "kind": "metric", "score": 0.5, "reason": "saw postgresql://harness:pw123@host/db"}], + call=None, + failure={"domain": "agent", "stage": "running", "code": "c", "message": "postgresql://harness:pw123@host/db"}, + ) + assert "pw123" not in str(receipt) + assert receipt["sub_goals"][0]["reason"] == "leaked postgresql://harness:***@host/db" + assert receipt["evaluations"][0]["reason"] == "saw postgresql://harness:***@host/db" + assert receipt["failure"]["message"] == "postgresql://harness:***@host/db" + # The digest must match the redacted bytes. + ResultReceiptDraft.model_validate(receipt) # round-trips cleanly, digest agrees + + +def test_redact_outbound_text_scrubs_userinfo_and_extra_secrets() -> None: + assert redact_outbound_text("postgresql://harness:pw@localhost/db") == "postgresql://harness:***@localhost/db" + assert redact_outbound_text("no secrets here") == "no secrets here" + assert redact_outbound_text("token=abc123 leaked", extra_secret_values=("abc123",)) == "token=*** leaked" + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + # P3: the empty-username shape -- the canonical Redis/RabbitMQ/Mongo DSN form -- used to + # leave the password verbatim (the old pattern required a NON-empty username group). + ("redis://:secretpw@h:6379/0", "redis://:***@h:6379/0"), + # A password containing '@' must not truncate the mask at the first '@' seen. + ("mysql://u:p@ss@h/db", "mysql://u:***@h/db"), + # A bare token/username-only userinfo (no ':') is the standard shape a bearer token takes + # in git/registry output -- must be masked outright, not treated as "just a username." + ("https://ghp_TOKEN@github.com/o/r.git", "https://***@github.com/o/r.git"), + # Negative: a path segment containing '@' with no userinfo before it must be left alone. + ("https://example.com/a@b", "https://example.com/a@b"), + # Negative: no "://" at all -- not a userinfo shape. + ("mailto:a@b", "mailto:a@b"), + ], +) +def test_redact_outbound_text_handles_the_empty_username_and_bare_token_userinfo_shapes( + raw: str, expected: str +) -> None: + assert redact_outbound_text(raw) == expected + + +# --- ArtifactsClient ----------------------------------------------------------------------------- + + +def test_artifacts_client_rejects_a_locally_wrong_digest_before_any_request() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + with pytest.raises(ValueError, match="artifact_digest_mismatch_local"): + client.upload("0" * 64, b"hello world", kind=ArtifactKind.LOG) + assert platform.calls == [] + + +def test_artifacts_client_upload_new_then_already_exists() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"result contents" + digest_hex = hashlib.sha256(data).hexdigest() + + first = client.upload(digest_hex, data, kind=ArtifactKind.RESULT, scenario_key="k1") + assert first.delivered and not first.already_existed + second = client.upload(digest_hex, data, kind=ArtifactKind.RESULT, scenario_key="k1") + assert second.delivered and second.already_existed + + +def test_artifacts_client_digest_mismatch_gets_exactly_one_re_upload() -> None: + platform = _platform() + platform.queue(TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {})) + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"re-upload me" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert result.delivered + assert len(platform.calls) == 2 # the queued failure, then the one allowed re-upload + + +def test_artifacts_client_digest_mismatch_twice_is_permanent_the_scenario_is_errored() -> None: + platform = _platform() + for _ in range(2): + platform.queue(TransportResponse(422, {"error": "digest_mismatch", "message": "m", "retryable": False}, {})) + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"re-upload me twice" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert not result.delivered + assert result.error is not None and result.error.code == "digest_mismatch" + assert len(platform.calls) == 2 # exactly one re-upload attempted, per "re-upload once" + + +def test_artifacts_client_sends_x_artifact_size_derived_from_the_actual_bytes() -> None: + # "X-Artifact-Size (authoritative; mismatch -> 422 size_mismatch)": this client never accepts a + # caller-supplied size that could drift from the real payload -- it always derives the header + # from `len(data)` itself, so a size mismatch against what's actually sent is structurally + # unreachable from this client (the platform's own check remains the authority regardless). + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"exactly this many bytes" + digest_hex = hashlib.sha256(data).hexdigest() + + client.upload(digest_hex, data, kind=ArtifactKind.TRACE, scenario_key="k9") + assert platform.received_headers[-1]["X-Artifact-Size"] == str(len(data)) + assert platform.received_headers[-1]["X-Artifact-Kind"] == "trace" + assert platform.received_headers[-1]["X-Scenario-Key"] == "k9" + + +def test_artifacts_client_content_type_defaults_by_kind() -> None: + # N17: recordings mp4, transcript a JSON array (per §3a "Formats") -- not a blanket octet-stream. + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"recording bytes" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.RECORDING_COMBINED) + assert platform.received_headers[-1]["Content-Type"] == "video/mp4" + + data2 = b"transcript bytes" + digest2 = hashlib.sha256(data2).hexdigest() + client.upload(digest2, data2, kind=ArtifactKind.TRANSCRIPT) + assert platform.received_headers[-1]["Content-Type"] == "application/json" + + data3 = b"trace bytes" + digest3 = hashlib.sha256(data3).hexdigest() + client.upload(digest3, data3, kind=ArtifactKind.TRACE) + assert platform.received_headers[-1]["Content-Type"] == "application/octet-stream" + + +def test_artifacts_client_content_type_override() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"custom type" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.OTHER, content_type="text/plain") + assert platform.received_headers[-1]["Content-Type"] == "text/plain" + + +def test_artifacts_client_latches_after_413_and_skips_non_reserved_without_the_transport() -> None: + # N18: once a 413 artifact_budget_exceeded is observed, later non-reserved uploads must be + # refused LOCALLY (never touching the transport again); reserved kinds keep going. + platform = _platform() + platform.budget_remaining = 1 + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + first_data = b"too big for the tiny budget" + first = client.upload(hashlib.sha256(first_data).hexdigest(), first_data, kind=ArtifactKind.TRACE) + assert first.error is not None and first.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + calls_after_413 = len(platform.calls) + + second_data = b"another non-reserved upload" + second = client.upload(hashlib.sha256(second_data).hexdigest(), second_data, kind=ArtifactKind.LOG) + assert not second.delivered + assert second.error is not None and second.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert len(platform.calls) == calls_after_413 # no new transport call + + reserved_data = b"reserved kind always goes through" + reserved_digest = hashlib.sha256(reserved_data).hexdigest() + third = client.upload(reserved_digest, reserved_data, kind=ArtifactKind.RESULT) + assert third.delivered # reserved kinds are never latched out + assert len(platform.calls) == calls_after_413 + 1 + + +def test_artifacts_client_413_budget_exceeded_is_never_retried() -> None: + platform = _platform() + platform.budget_remaining = 5 + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"too big for the budget" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.LOG) + assert not result.delivered + assert result.error is not None + assert result.error.outcome is ChannelOutcome.BUDGET_EXCEEDED + assert len(platform.calls) == 1 + + +def test_artifacts_client_chunks_uploads_over_the_threshold_and_reassembles_correctly() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None, chunk_threshold_bytes=10, chunk_size_bytes=4) + data = b"0123456789abcdef" + digest_hex = hashlib.sha256(data).hexdigest() + + result = client.upload(digest_hex, data, kind=ArtifactKind.TRACE) + assert result.delivered + assert platform._artifacts[digest_hex] == data + + +def test_artifacts_client_crash_between_send_and_ack_redelivers_as_already_exists() -> None: + platform = _platform() + data = b"artifact bytes" * 100 + digest_hex = hashlib.sha256(data).hexdigest() + platform.crash_after_next_write = True + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + + result = client.upload(digest_hex, data, kind=ArtifactKind.TRACE) + assert result.delivered + assert result.already_existed + assert len(platform.calls) == 2 + assert platform._artifacts[digest_hex] == data + + +def test_artifacts_client_push_manifest_is_idempotent_and_checks_references() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + data = b"manifest target" + digest_hex = hashlib.sha256(data).hexdigest() + client.upload(digest_hex, data, kind=ArtifactKind.RESULT) + + manifest = build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": f"sha256:{digest_hex}", "kind": "result", "size": len(data), "scenario_key": "k"}], + complete=True, + ) + first = client.push_manifest(manifest) + assert first.delivered and not first.already_existed + second = client.push_manifest(manifest) + assert second.delivered and second.already_existed # idempotent on (attempt_id, digest) + + +def test_artifacts_client_push_manifest_rejects_an_unknown_artifact_reference() -> None: + platform = _platform() + client = ArtifactsClient(_capabilities(), platform, sleep=lambda s: None) + manifest = build_artifact_manifest( + job_id="j1", attempt_id="a1", attempt_number=1, + entries=[{"artifact_id": "sha256:" + "f" * 64, "kind": "result", "size": 1, "scenario_key": "k"}], + complete=True, + ) + result = client.push_manifest(manifest) + assert not result.delivered + assert result.error is not None and result.error.code == "artifact_unknown" diff --git a/tests/harness/test_process_preflight.py b/tests/harness/test_process_preflight.py new file mode 100644 index 00000000..1b1cc187 --- /dev/null +++ b/tests/harness/test_process_preflight.py @@ -0,0 +1,1103 @@ +"""The §2e pre-provision checklist (`process_preflight.py`), per `hosted-execution-seams.md` v1.9. + +Every checklist item gets at least one rejection test carrying its named code, plus one clean +accept-lane run of the full checklist. Bundles are built as real directories under `tmp_path` with +real file bytes — the digest and per-file checks need actual content to hash, so a manifest built +purely in memory (as `test_bundle_v2.py` does) cannot exercise this module. No docker; every +managed-engine/process concept here is a manifest fact, never a running service. +""" + +from __future__ import annotations + +import ast +import hashlib +import inspect +import json +import re +from pathlib import Path +from typing import Any, Callable + +import pytest + +from fi.alk.harness import bundle as bundle_module +from fi.alk.harness import bundle_v2 as bundle_v2_module +from fi.alk.harness import process_preflight as process_preflight_module +from fi.alk.harness import process_runtime as process_runtime_module +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + EnvironmentBundleV2, + ManagedEngine, + compute_inputs_digest, + seal_bundle_v2, +) +from fi.alk.harness.process_preflight import PreflightError, preflight_bundle + +SCHEMA_SQL = b"CREATE TABLE riders (id int);\n" +SEED_SQL = b"INSERT INTO riders VALUES (1);\n" + +TARGET_PROVIDER_REFS = {"LIVEKIT_API_KEY": "target_provider"} + + +def _base_manifest_body() -> dict[str, Any]: + """A minimal, otherwise-clean `kind: process` manifest: one postgres store behind a real + seeded capability, one source process that claims the job's only `target_provider` secret.""" + return { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", + "kind": "managed", + "engine": "postgres", + "version": "16", + "user": "svc-data", + "depends_on": [], + }, + { + "name": "agent", + "kind": "source", + "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", + "LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], + "user": "svc-agent", + "depends_on": ["postgres"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "DATABASE_URL", + }, + }, + "readiness": [], + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64 + }, + "metadata": {}, + } + + +def _build_bundle( + root: Path, + *, + body_overrides: Callable[[dict[str, Any]], dict[str, Any]] | None = None, + extra_files: dict[str, bytes] | None = None, + unlisted_files: dict[str, bytes] | None = None, + include_seed: bool = True, +) -> EnvironmentBundleV2: + """Write a real, digest-consistent bundle directory and return its parsed manifest, sealed + through `seal_bundle_v2` — the module under test's own producer, not a second reimplementation + of it (the accept lane must prove this module agrees with a real producer, not just with + itself). ``unlisted_files`` writes real bytes to disk without ever hashing them into + ``files[]`` — the shape a producer bug or a stray leftover file takes, as opposed to + ``extra_files``, which is hashed in and fully listed.""" + root.mkdir(parents=True, exist_ok=True) + body = _base_manifest_body() + file_contents = {"db/schema.sql": SCHEMA_SQL, "db/seed.sql": SEED_SQL} + if extra_files: + file_contents.update(extra_files) + + files: list[dict[str, Any]] = [] + for relative, content in file_contents.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + files.append( + {"path": relative, "sha256": hashlib.sha256(content).hexdigest(), "size": len(content)} + ) + body["files"] = files + + if include_seed: + digest = compute_inputs_digest( + root, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + body["seed"] = { + "stores": [ + { + "capability": "database", + "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": digest}, + "sentinel": {"query": "SELECT count(*) FROM riders", "expected": "1"}, + } + ] + } + + if body_overrides is not None: + body = body_overrides(body) + + body["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(body) + body["digest"] = seal_bundle_v2(normalized) + + if unlisted_files: + for relative, content in unlisted_files.items(): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + (root / "manifest.json").write_text(json.dumps(body, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(body) + + +# --- accept lane -------------------------------------------------------------------------------- + + +def test_a_clean_bundle_passes_the_whole_checklist(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + result = preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +# --- item 1: digest verification ---------------------------------------------------------------- + + +def test_a_file_whose_bytes_changed_after_sealing_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").write_bytes(b"MUTATED") + with pytest.raises(PreflightError, match="bundle_file_changed") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_file_changed" + + +def test_a_files_entry_missing_from_disk_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").unlink() + with pytest.raises(PreflightError, match="bundle_file_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_bundle_digest_that_does_not_match_the_recomputed_value_is_rejected( + tmp_path: Path, +) -> None: + manifest = _build_bundle(tmp_path) + tampered = manifest.model_copy(update={"digest": "sha256:" + "9" * 64}) + with pytest.raises(PreflightError, match="bundle_digest_mismatch"): + preflight_bundle(tmp_path, tampered, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 2: path safety on the filesystem -------------------------------------------------------- + + +def test_a_symlink_anywhere_under_the_bundle_is_rejected(tmp_path: Path) -> None: + """The model already forbids unsafe strings in `files[].path`; this is the filesystem-level + complement — a symlink is rejected even when it sits outside `files[]` entirely.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "evil-link").symlink_to(tmp_path / "db" / "schema.sql") + with pytest.raises(PreflightError, match="bundle_symlink_forbidden"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_root_manifest_json_itself_being_a_symlink_is_rejected(tmp_path: Path) -> None: + """N3 (p4-round2-review): the root `manifest.json` is exempt from the listing check (a + manifest cannot list itself), but that exemption must not extend to its symlink status — a + symlinked root manifest was previously read straight through by `_verify_digest` and + `_verify_unknown_fields`, the very item whose job is to stop path escapes.""" + manifest = _build_bundle(tmp_path) + outside = tmp_path.parent / f"{tmp_path.name}-manifest-target.json" + outside.write_bytes((tmp_path / "manifest.json").read_bytes()) + (tmp_path / "manifest.json").unlink() + (tmp_path / "manifest.json").symlink_to(outside) + with pytest.raises(PreflightError, match="bundle_symlink_forbidden"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_file_present_on_disk_but_not_listed_in_files_is_rejected(tmp_path: Path) -> None: + """F1 (p4-round1-review): a bundle directory containing a file the producer never hashed into + `files[]` was invisible to both the digest check and the secret scan — this is the case that + previously slipped a `.env` through undetected. `extra_files` (used by the item-3 tests below) + would have hashed it in; `unlisted_files` writes the same bytes without listing them at all.""" + manifest = _build_bundle(tmp_path, unlisted_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="bundle_file_unlisted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 3: secret material in the bundle's own files ------------------------------------------- + + +def test_a_dotenv_file_in_the_bundle_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path, extra_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_high_entropy_secret_content_in_a_bundle_file_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle( + tmp_path, extra_files={"db/notes.sql": b"-----BEGIN RSA PRIVATE KEY-----\nabc\n"} + ) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 4: unknown-field translation ------------------------------------------------------------ + + +def test_an_unknown_field_added_to_the_manifest_on_disk_is_rejected(tmp_path: Path) -> None: + """The `manifest` argument is already-parsed and therefore already clean; this proves the + translation fires against the bytes on disk, which is what a drifted or hand-edited + `manifest.json` would look like to a fresh `EnvironmentBundleV2.model_validate` call.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["mounts"] = ["/data"] + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="unknown_field") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "unknown_field" + + +def test_a_valid_but_drifted_manifest_on_disk_is_rejected(tmp_path: Path) -> None: + """F12 (p4-round1-review): re-validating the bytes on disk only catches drift that makes the + file *invalid* — this proves drift that leaves it valid (a changed `run_command`) is caught + too, by actually comparing the two dumps rather than discarding the re-validated one.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["processes"][1]["run_command"] = ["python", "other.py"] + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_manifest_drifted") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_manifest_drifted" + + +def test_an_unreadable_manifest_json_on_disk_is_rejected(tmp_path: Path) -> None: + """F18 (p4-round1-review): `bundle_manifest_invalid`'s parse-failure branch, untested before.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "manifest.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_manifest_invalid"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_non_extra_forbidden_model_rejection_surfaces_its_own_code(tmp_path: Path) -> None: + """F13/F18 (p4-round1-review): `_translate_validation_error`'s fallback path was untested. A + malformed on-disk `digest` (item 1 never reads it — only the `manifest` argument's own valid + digest and file hashes, which are untouched here) reaches item 4's re-validation and raises + `bundle_digest_invalid`, a bare code with no trailing colon at all — exactly the case the old + `:`-only regex flattened to the generic `bundle_manifest_invalid`.""" + manifest = _build_bundle(tmp_path) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + raw["digest"] = "not-a-valid-digest" + (tmp_path / "manifest.json").write_text(json.dumps(raw), encoding="utf-8") + with pytest.raises(PreflightError, match="bundle_digest_invalid") as excinfo: + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert excinfo.value.code == "bundle_digest_invalid" + + +# --- §2a: compose is not a legal hosted runtime kind ---------------------------------------------- + + +def test_kind_compose_is_rejected(tmp_path: Path) -> None: + """N1 (p4-round2-review): the gate sits above item 1, so this fires regardless of whether + `compose.yaml` exists on disk or is listed in `files[]` — a compose bundle need carry neither + to be rejected, which is why this fixture writes no such file at all.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +# --- item 5: placeholder vocabulary, secret purposes, depends_on, engine catalog, interpreter, +# seed_missing, reserved names, build_requires_root, seed files on disk+listed --------------------- + + +def test_a_placeholder_outside_the_closed_vocabulary_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_port_placeholder_naming_an_unknown_process_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["PEER_PORT"] = "{{PORT_ghost-service}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_port_placeholder_with_an_empty_name_falls_through_to_unknown_placeholder( + tmp_path: Path, +) -> None: + """F18 (p4-round1-review): `(.+)` in `_NAMED_PLACEHOLDER` requires at least one character after + `PORT_`/`HOST_`, so `{{PORT_}}` does not match the named-placeholder pattern at all — it falls + through to the configuration-name set, misses, and is rejected as `unknown_placeholder`. A + plausible regression target if `(.+)` is ever relaxed to `(.*)`.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["EMPTY"] = "{{PORT_}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_token_naming_a_capability_with_no_configuration_name_is_capability_unresolved( + tmp_path: Path, +) -> None: + """F15 (p4-round1-review): a null `configuration_name` is structurally unspellable by any + placeholder — but when the unmatched token happens to be exactly a declared capability's own + slug, the real problem is the capability's missing name, not an unrecognized token, so this is + reported `capability_unresolved` naming the capability rather than the generic + `unknown_placeholder`.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["capabilities"]["cache"] = {"protocol": "http", "service": "postgres"} + body["processes"][1]["environment"]["FOO"] = "{{cache}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="capability_unresolved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_build_environment_rejects_any_placeholder_even_a_legal_one(tmp_path: Path) -> None: + """F6 (p4-round1-review): §2b's `build_environment` takes NO placeholders at all — this uses + `{{WORLD_DIR}}`, a perfectly legal token in `environment`, specifically because that is the + case a naive "scan against the same vocabulary" implementation would wave through.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_environment"] = {"TMPDIR": "{{WORLD_DIR}}"} + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_fixed_and_named_placeholders_are_accepted(tmp_path: Path) -> None: + """`{{WORLD_INDEX}}`/`{{WORLD_DIR}}`/`{{DB_NAME}}` need no lookup; `{{PORT_}}` and + `{{HOST_}}` need only a real process name — neither needs a capability.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["SCRATCH"] = "{{WORLD_DIR}}" + body["processes"][1]["environment"]["DB"] = "{{DB_NAME}}" + body["processes"][1]["environment"]["PEER"] = "{{HOST_postgres}}:{{PORT_postgres}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + result = preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +def test_a_build_command_requiring_root_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_commands"] = [["apt-get", "install", "-y", "ffmpeg"]] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="build_requires_root"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_build_command_with_sudo_anywhere_in_the_step_is_rejected(tmp_path: Path) -> None: + """F18 (p4-round1-review): only `apt-get` as argv[0] was exercised before — `"sudo" in step` + is exact-token list membership, not a substring match, so `sudo` appearing anywhere in the + argv list (not just as argv[0]) must trip it too.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["build_commands"] = [["scripts/setup.sh", "--with-sudo", "sudo"]] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="build_requires_root"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_target_provider_ref_no_process_lists_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["secret_purposes"] = [] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="secret_unclaimed"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_listed_secret_purpose_with_no_supplying_ref_is_rejected(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + with pytest.raises(PreflightError, match="secret_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +def test_omitting_secret_refs_is_a_typeerror(tmp_path: Path) -> None: + """F2 (p4-round1-review): `secret_refs` is required, not optional with an empty-dict default — + an optional default meant `secret_unclaimed` was structurally unreachable for any caller that + relied on it, while `secret_missing` fired on bundles the default should have let alone. This + pins the signature itself: omitting the argument must fail loudly, at the call site, rather + than silently reintroducing the empty-dict default.""" + manifest = _build_bundle(tmp_path) + with pytest.raises(TypeError): + preflight_bundle(tmp_path, manifest, parallelism=1) # type: ignore[call-arg] + + +def test_an_unrecognized_secret_purpose_value_is_rejected(tmp_path: Path) -> None: + """F2 (p4-round1-review), the signature's second defect: `secret_refs` values are only ever + compared by `==` against `SecretPurpose.TARGET_PROVIDER.value` — a typo'd purpose string (or + §1's raw per-alias dict shape, whose `purpose` values are dicts, not strings) silently never + matches, producing the same wrong verdict as the missing-validation default did. Validating + eagerly turns that into a loud, immediate `ValueError`.""" + manifest = _build_bundle(tmp_path) + with pytest.raises(ValueError, match="not a SecretPurpose"): + preflight_bundle( + tmp_path, manifest, parallelism=1, secret_refs={"LIVEKIT_API_KEY": "target-provider"} + ) + + +def test_a_depends_on_naming_an_unknown_process_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["depends_on"] = ["postgres", "ghost"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="depends_on_unresolved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_depends_on_cycle_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][0]["depends_on"] = ["agent"] + body["processes"][1]["depends_on"] = ["postgres"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="depends_on_cycle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_an_engine_version_outside_the_catalog_pin_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][0]["version"] = "15" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="engine_unsupported"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +@pytest.mark.parametrize("colliding_port", [14000, 14099, 15000, 15799]) +def test_a_fixed_port_colliding_with_a_port_formula_band_is_rejected( + tmp_path: Path, colliding_port: int +) -> None: + """F11, p5-round1-review: `fixed_port` forces effective parallelism to 1, but the literal + value was never checked against the provisioner's own port-formula bands — a bundle declaring + `fixed_port: 14000` collides with a job-shared engine at ordinal 0, and the failure mode is an + opaque bind error inside a customer process, not a bundle rejection. `fixed_port_reserved` is + in §2e's closed failure-code table as of v1.9.""" + + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = colliding_port + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="fixed_port_reserved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +@pytest.mark.parametrize("colliding_port", [24000, 24099, 25000, 25799]) +def test_a_fixed_port_colliding_with_the_rabbitmq_management_band_is_rejected( + tmp_path: Path, colliding_port: int +) -> None: + """M4: `process_runtime.py`'s own rabbitmq management listener binds at `amqp_port + 10000` + (`_rabbitmq_management_port`) -- a `fixed_port` landing there is not covered by either base + port-formula band, so a bundle could claim a port the harness itself is about to bind. Bands: + 24000-24099 (job-shared amqp shifted) and 25000-25799 (per-world amqp shifted, the one + reachable in practice since rabbitmq's catalog entry is `datadir_copy`-only).""" + + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = colliding_port + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="fixed_port_reserved"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_fixed_port_outside_both_bands_is_accepted(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["fixed_port"] = 9000 + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + assert preflight_bundle( + tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS + ) is None + + +def test_a_postgres_capability_with_no_store_entry_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["capabilities"]["other_db"] = { + "protocol": "postgres", + "service": "postgres", + "configuration_name": "OTHER_DB_URL", + } + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="seed_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def _reseal_schema_sql(tmp_path: Path, content: bytes) -> EnvironmentBundleV2: + """Rewrites `db/schema.sql` on disk and reseals through `seal_bundle_v2` — so digest + verification (item 1, which runs before item 5's content-scanning checks) passes on the + mutated content, isolating a test to the check the mutation is actually meant to exercise.""" + (tmp_path / "db" / "schema.sql").write_bytes(content) + raw = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) + for record in raw["files"]: + if record["path"] == "db/schema.sql": + record["sha256"] = hashlib.sha256(content).hexdigest() + record["size"] = len(content) + raw["seed"]["stores"][0]["baseline"]["inputs_digest"] = compute_inputs_digest( + tmp_path, ["db/schema.sql"], ["db/seed.sql"], engine=ManagedEngine.POSTGRES, version="16" + ) + raw["digest"] = "sha256:" + "0" * 64 + normalized = EnvironmentBundleV2.model_validate(raw) + raw["digest"] = seal_bundle_v2(normalized) + (tmp_path / "manifest.json").write_text(json.dumps(raw, indent=2), encoding="utf-8") + return EnvironmentBundleV2.model_validate(raw) + + +def test_the_reserved_conformance_name_in_migration_content_is_rejected(tmp_path: Path) -> None: + _build_bundle(tmp_path) + manifest = _reseal_schema_sql(tmp_path, b"CREATE TABLE _alk_conformance (id int);\n") + with pytest.raises(PreflightError, match="reserved_name"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_the_reserved_conformance_name_is_matched_case_insensitively(tmp_path: Path) -> None: + """F9 (p4-round1-review): postgres folds an unquoted identifier to lower case, so + `CREATE TABLE _ALK_CONFORMANCE` creates the reserved table under its lower-case name — a + case-sensitive scan would miss exactly the evasion this exists to catch.""" + _build_bundle(tmp_path) + manifest = _reseal_schema_sql(tmp_path, b"CREATE TABLE _ALK_CONFORMANCE (id int);\n") + with pytest.raises(PreflightError, match="reserved_name"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_similarly_named_table_and_a_comment_mentioning_the_reserved_name_are_accepted( + tmp_path: Path, +) -> None: + """F9 (p4-round1-review) regression: `alk_conformance_backup` (no leading underscore, and a + different table) must not trip the scan — this is exactly what the lookarounds exist to keep + open. A `--` comment mentioning the reserved name as prose, not as an identifier it defines, + must not trip it either, now that comments are stripped before scanning.""" + _build_bundle(tmp_path) + manifest = _reseal_schema_sql( + tmp_path, + b"CREATE TABLE alk_conformance_backup (id int);\n" + b"-- never create _alk_conformance here\n", + ) + result = preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + assert result is None + + +def test_a_migration_path_not_listed_in_files_is_rejected(tmp_path: Path) -> None: + """`seed_file_unlisted` (item 5) and `bundle_file_unlisted` (item 2, F1 p4-round1-review) share + the same "on disk but not in files[]" condition — item 2's bundle-wide walk now runs first and + always wins this exact scenario, which is why this asserts the earlier-numbered item's code + rather than the one this test used to name before F1 closed item 2's gap.""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["migrations"] = ["db/schema.sql", "db/extra.sql"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + # On disk, but never hashed into files[]. + (tmp_path / "db" / "extra.sql").write_bytes(b"-- extra\n") + with pytest.raises(PreflightError, match="bundle_file_unlisted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_seed_file_path_that_does_not_exist_on_disk_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["seed_files"] = ["db/seed.sql", "db/ghost.sql"] + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="seed_file_missing"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_recorded_inputs_digest_that_does_not_match_the_seed_files_is_rejected( + tmp_path: Path, +) -> None: + """F14 (p4-round1-review): §2c makes `inputs_digest` the baseline identity attempt-retry reuse + trusts absolutely — nothing on either side of the seam validated it before. `engine`/`version` + come from the store's capability's own backing `ManagedProcess` (postgres/16 here).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["seed"]["stores"][0]["baseline"]["inputs_digest"] = "sha256:" + "f" * 64 + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="inputs_digest_mismatch"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 6: store / evidence-seam coherence ----------------------------------------------------- + + +def _storeless(body: dict[str, Any]) -> dict[str, Any]: + """Zero postgres-protocol capabilities. Keeps the `database` capability slug (so the + `{{DATABASE_URL}}` placeholder in `agent`'s environment still resolves) and only changes its + protocol away from postgres, isolating these tests from the placeholder-vocabulary check that + would otherwise fire first.""" + body["capabilities"]["database"]["protocol"] = "http" + body["seed"] = None + return body + + +def test_a_storeless_process_bundle_passes_preflight(tmp_path: Path) -> None: + # A stateless agent is legal: nothing else in the checklist needs a store, and the runtime's + # storeless world only fails, by type, when scenario code actually asks it for state. + manifest = _build_bundle(tmp_path, body_overrides=_storeless, include_seed=False) + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_storeless_bundle_declaring_tool_trace_evidence_is_rejected(tmp_path: Path) -> None: + # `tool_trace` reads the agent's tool calls out of the world's postgres store — with no store + # the seam can never produce evidence, so it must fail here, not as `evidence_missing` on + # every scenario at runtime. + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body = _storeless(body) + body["runtime"]["evidence_seam"] = "tool_trace" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate, include_seed=False) + with pytest.raises(PreflightError, match="evidence_seam_unsatisfiable"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +# --- item 7: resource sanity --------------------------------------------------------------------- + + +def test_more_than_100_processes_is_rejected(tmp_path: Path) -> None: + def mutate(body: dict[str, Any]) -> dict[str, Any]: + extra = [ + { + "name": f"extra-{i}", + "kind": "managed", + "engine": "redis", + "version": "7", + "user": "svc-data", + "depends_on": [], + } + for i in range(100) + ] + body["processes"] = body["processes"] + extra + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="process_count_exceeded"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +@pytest.mark.parametrize("parallelism", [0, 9], ids=["below-range", "above-range"]) +def test_parallelism_outside_1_to_8_is_rejected(tmp_path: Path, parallelism: int) -> None: + manifest = _build_bundle(tmp_path) + with pytest.raises(PreflightError, match="parallelism_out_of_range"): + preflight_bundle( + tmp_path, manifest, parallelism=parallelism, secret_refs=TARGET_PROVIDER_REFS + ) + + +# --- ordering: the checklist runs in the contract's numbered order, and a bundle violating two +# rules at once must report the earlier-numbered one (F17, p4-round1-review) — every rejection +# fixture above proves presence, none of them alone proves position. -------------------------- + + +def test_a_secret_file_and_an_unknown_placeholder_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 3 (secret scan) before item 5 (placeholder vocabulary).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate, extra_files={".env": b"SECRET=1\n"}) + with pytest.raises(PreflightError, match="secret_in_bundle"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_changed_file_and_a_reserved_name_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 1 (digest verification) before item 5's reserved-name scan — the mutated file carries + both a byte-level tamper the digest catches and a reserved name the content scan would catch, + but the manifest is never resealed, so item 1 sees it first.""" + manifest = _build_bundle(tmp_path) + (tmp_path / "db" / "schema.sql").write_bytes(b"CREATE TABLE _alk_conformance (id int);\n") + with pytest.raises(PreflightError, match="bundle_file_changed"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_placeholder_and_bad_parallelism_together_report_the_earlier_numbered_item( + tmp_path: Path, +) -> None: + """Item 5 (placeholder vocabulary) before item 7 (resource sanity).""" + def mutate(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["FOO"] = "{{NOT_A_REAL_TOKEN}}" + return body + + manifest = _build_bundle(tmp_path, body_overrides=mutate) + with pytest.raises(PreflightError, match="unknown_placeholder"): + preflight_bundle(tmp_path, manifest, parallelism=99, secret_refs=TARGET_PROVIDER_REFS) + + +def test_a_compose_bundle_with_an_unlisted_file_reports_compose_not_hosted( + tmp_path: Path, +) -> None: + """N1 (p4-round2-review): the compose gate before item 1, before item 2 (`test_kind_compose_ + is_rejected` alone cannot prove position, since it now carries no file to trip item 2 at all). + An unlisted `compose.yaml` would report `bundle_file_unlisted` if item 2 ran first — this pins + that the gate wins instead.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + (tmp_path / "compose.yaml").write_bytes(b"services: {}\n") + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +def test_a_compose_bundle_with_a_changed_listed_file_reports_compose_not_hosted( + tmp_path: Path, +) -> None: + """B2 (p4-round3-review): the fixture above only pins the gate above item 2 — its sole + competing violation (`compose.yaml` unlisted) never reaches item 1, since the bundle is + otherwise digest-consistent. This mutates a LISTED file's bytes after sealing, without + resealing, so item 1 would otherwise raise `bundle_file_changed` — proving the gate wins + above item 1 too, not just item 2.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "compose", "document": "compose.yaml"}, + "processes": [], + "seed": None, + }, + include_seed=False, + ) + (tmp_path / "compose.yaml").write_bytes(b"services: {}\n") + (tmp_path / "db" / "schema.sql").write_bytes(b"MUTATED") # still listed in files[]; unsealed. + with pytest.raises(PreflightError, match="compose_not_hosted"): + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs={}) + + +# --- kind: external ------------------------------------------------------------------------------ + + +def test_kind_external_skips_the_process_block_but_still_enforces_resource_sanity( + tmp_path: Path, +) -> None: + """F18 (p4-round1-review): the only prior `external` coverage was model-layer + (`test_bundle_v2.py`) — nothing proved that `preflight_bundle` itself skips item 5's process + block and item 6 (`no_sql_store`) for `kind: external`, and still applies item 7.""" + manifest = _build_bundle( + tmp_path, + body_overrides=lambda b: { + **b, + "runtime": {"kind": "external"}, + "processes": [], + "seed": None, + "capabilities": { + "target": { + "protocol": "http", + "service": "customer-endpoint", + "configuration_name": "TARGET_URL", + }, + }, + }, + include_seed=False, + ) + # Item 5 (placeholder/secret-purpose/depends_on/engine-catalog/seed checks) and item 6 + # (no_sql_store) never run — nothing in this manifest could satisfy either, since it has no + # processes at all. + assert preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs={}) is None + # Item 7 still runs regardless of runtime kind. + with pytest.raises(PreflightError, match="parallelism_out_of_range"): + preflight_bundle(tmp_path, manifest, parallelism=99, secret_refs={}) + + +# --- §2e-table containment (B3/S3, p4-round3-review) --------------------------------------------- +# +# `PreflightError`'s own docstring claims "every code this module raises is in that [§2e] table" — +# true today, per p4-round3-review's part-B finding, but mechanically unenforced: a future +# `raise ValueError("some_new_code: ...")` anywhere in the model layer would silently leak an +# unlisted code across a seam the contract calls closed, and nothing short of re-deriving the claim +# by hand (as that review did) would catch it. This makes the claim a running test instead. +# +# Approach (grep-free, as the worklist asked): walk each module's SOURCE TEXT with `ast`, not +# `grep` — a plain substring/regex scan over raw text can't distinguish an actual `raise +# PreflightError(...)`/`raise ValueError(...)` call from a code fragment inside a docstring or a +# comment quoting one. Two extraction shapes: +# - `PreflightError(...)`: the whole first positional argument, when it is a literal string, IS +# the code (every call in this codebase spells it that way) — `_translate_validation_error`'s +# `extra_forbidden` branch RETURNS its `PreflightError(...)` rather than raising it directly +# (the caller does `raise _translate_validation_error(exc) from exc`), so the walk matches +# every `Call` node by callee name, not only ones sitting directly under a `Raise`. +# - `ValueError(...)` (the model layer): the code is only the LEADING token of the message, up +# to `:` or end-of-string — recovered by taking the static leading text of the first argument +# (a plain string constant, an f-string's first literal segment, or the left operand of a `+` +# concatenation — every shape actually used in `bundle_v2.py`/`bundle.py`) and applying the +# same `[a-z][a-z0-9_]*` leading-token regex `_translate_validation_error`'s own runtime +# fallback uses. A `ValueError` whose message has no static leading text at all (fully +# dynamic) would be skipped rather than mis-coded — none exist in these three modules today. +# +# Scope judgement call: `bundle.py` also carries `EnvironmentBundle`/`BundleRuntime` (v1's OWN, +# separate model classes) with their own, unrelated `ValueError` raises (e.g. plain prose with no +# code prefix at all, `bundle_digest_invalid`/`bundle_schema_unsupported` reused as v1's own +# names) — none of that is reachable from `EnvironmentBundleV2.model_validate(...)`'s validation +# tree, since v1 and v2 are disjoint model hierarchies (`bundle_v2.py`'s own docstring: "v1 stays +# untouched"). The only `bundle.py` surface v2 actually calls into is the two helper functions +# `bundle_v2.py` imports and its own validators invoke — `_safe_relative`, `_reject_secret_values` +# — so this test walks exactly those two functions' source (`inspect.getsource`), not the whole +# file; walking the whole file would both over-include unreachable v1 codes and risk masking a +# real gap behind noise from a hierarchy this claim was never about. + + +_CODE_LEADING_TOKEN = re.compile(r"^([a-z][a-z0-9_]*)(?::|$)") + + +def _static_leading_text(node: ast.expr) -> str | None: + """The static leading text of a raise-argument expression, or `None` if it has none at all + (a fully dynamic value, e.g. a bare name or an f-string starting with a `{...}`).""" + if isinstance(node, ast.Constant) and isinstance(node.value, str): + return node.value + if isinstance(node, ast.JoinedStr) and node.values and isinstance(node.values[0], ast.Constant): + return str(node.values[0].value) + if isinstance(node, ast.BinOp) and isinstance(node.op, ast.Add): + return _static_leading_text(node.left) + return None + + +def _raised_codes(source_text: str, callee_name: str, *, whole_first_argument: bool) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(source_text)): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Name) and func.id == callee_name): + continue + if not node.args: + continue + if whole_first_argument: + first = node.args[0] + if isinstance(first, ast.Constant) and isinstance(first.value, str): + codes.add(first.value) + continue + leading = _static_leading_text(node.args[0]) + if leading is None: + continue + match = _CODE_LEADING_TOKEN.match(leading) + if match: + codes.add(match.group(1)) + return codes + + +# §2e's closed failure-code table (v1.9), transcribed verbatim — the single source of truth every +# raised code is checked against. Split exactly as the contract text splits it, purely for +# reviewability against the spec; the test below treats it as one flat set. +_SECTION_2E_CONTRACT_RULE_CODES = frozenset({ + "compose_not_hosted", "engine_unsupported", "no_sql_store", "seed_missing", + "seed_strategy_unsupported", "sentinel_shape_mismatch", "store_protocol_unsupported", + "capability_engine_mismatch", "store_service_not_managed", "reserved_name", + "unknown_placeholder", "unknown_field", "secret_in_bundle", "secret_unclaimed", + "secret_missing", "build_requires_root", "user_assignment_invalid", + "configuration_name_duplicate", "configuration_name_required", + "configuration_name_reserved", "sentinel_shape_invalid", "capability_unresolved", + "service_unresolved", "control_service_unresolved", "process_name_duplicate", + "inputs_digest_mismatch", + "fixed_port_reserved", # §2e, v1.9. +}) +_SECTION_2E_MECHANICAL_CODES = frozenset({ + "agent_name_not_world_unique", + # storeless bundles: a bundle with no postgres capability cannot satisfy `evidence_seam: + # tool_trace` — the §2e contract table still needs the amendment adding this code. + "evidence_seam_unsatisfiable", + "bundle_schema_unsupported", "bundle_manifest_invalid", "bundle_manifest_drifted", + "bundle_digest_mismatch", "bundle_digest_invalid", "inputs_digest_invalid", + "file_sha256_invalid", "source_digest_invalid", "bundle_file_missing", + "bundle_file_changed", "bundle_file_unlisted", "bundle_symlink_forbidden", + "bundle_path_unsafe", "depends_on_unresolved", "depends_on_cycle", "seed_file_missing", + "seed_file_unlisted", "process_count_exceeded", "parallelism_out_of_range", + "evidence_seam_required", "processes_required", "processes_and_seed_forbidden", + "document_only_for_compose", "compose_runtime_requires_document", + "build_command_step_empty", "started_check_requires_exactly_one_of_port_or_log_marker", + "resolved_secret_forbidden", "capability_slug_invalid", + "process_name_invalid", # §0/§2b v1.8: the process-`name` pattern rule. +}) +_SECTION_2E_CODES = _SECTION_2E_CONTRACT_RULE_CODES | _SECTION_2E_MECHANICAL_CODES + + +def test_every_code_these_modules_can_raise_is_in_the_closed_section_2e_table() -> None: + raised: set[str] = set() + raised |= _raised_codes( + Path(inspect.getfile(process_preflight_module)).read_text(encoding="utf-8"), + "PreflightError", whole_first_argument=True, + ) + raised |= _raised_codes( + Path(inspect.getfile(bundle_v2_module)).read_text(encoding="utf-8"), + "ValueError", whole_first_argument=False, + ) + raised |= _raised_codes( + inspect.getsource(bundle_module._safe_relative), "ValueError", whole_first_argument=False + ) + raised |= _raised_codes( + inspect.getsource(bundle_module._reject_secret_values), + "ValueError", whole_first_argument=False, + ) + unlisted = raised - _SECTION_2E_CODES + assert not unlisted, f"raised but not in §2e's table: {sorted(unlisted)}" + + +def test_the_extraction_itself_finds_a_nonempty_set_in_every_source() -> None: + """Guards the test above against a false pass from a broken extractor (e.g. an import that + silently resolves to the wrong file, or a callee-name typo) — each source individually must + contribute at least one code, not just the union as a whole.""" + preflight_codes = _raised_codes( + Path(inspect.getfile(process_preflight_module)).read_text(encoding="utf-8"), + "PreflightError", whole_first_argument=True, + ) + bundle_v2_codes = _raised_codes( + Path(inspect.getfile(bundle_v2_module)).read_text(encoding="utf-8"), + "ValueError", whole_first_argument=False, + ) + assert "unknown_field" in preflight_codes # the `return`-not-`raise` case (see module note). + assert "compose_not_hosted" in preflight_codes + assert len(bundle_v2_codes) > 10 + assert "bundle_path_unsafe" in _raised_codes( + inspect.getsource(bundle_module._safe_relative), "ValueError", whole_first_argument=False + ) + assert "resolved_secret_forbidden" in _raised_codes( + inspect.getsource(bundle_module._reject_secret_values), + "ValueError", whole_first_argument=False, + ) + + +# --- §2f-table containment (v1.8 addition) -------------------------------------------------------- +# +# v1.8 gave `process_runtime.py` its own closed table (§2f) for the subset of `ProcessRuntimeError` +# codes that cross the outbound seam. Same containment argument as §2e above, extended to the +# module the new table covers — worklist item: `unsupported_capability_protocol` and +# `source_tree_unavailable` are new in this fix pass and must show up here or the test (correctly) +# fails. `ProcessRuntimeError(stage, code, message, ...)` carries its code at positional index 1, +# not index 0 (`PreflightError`'s shape), so this is a distinct extraction, not a reuse of +# `_raised_codes`. + + +def _raised_codes_at_index(source_text: str, callee_name: str, *, index: int) -> set[str]: + codes: set[str] = set() + for node in ast.walk(ast.parse(source_text)): + if not isinstance(node, ast.Call): + continue + func = node.func + if not (isinstance(func, ast.Name) and func.id == callee_name): + continue + if len(node.args) <= index: + continue + argument = node.args[index] + if isinstance(argument, ast.Constant) and isinstance(argument.value, str): + codes.add(argument.value) + return codes + + +# §2f's closed table (v1.8), transcribed verbatim, plus v1.10's two additions below. +_SECTION_2F_CODES = frozenset({ + "source_tree_unavailable", "build_failed", "runtime_unsupported", "spawn_failed", + "depends_on_timeout", "unsupported_capability_protocol", + # `seed_failed` (v1.10, §2f): a §2c migration/seed step exited nonzero against the freshly + # started store — customer-authored content, deterministic, `environment` domain (never + # retried). Landed in the frozen table this version; no longer an out-of-vocabulary flag. + "seed_failed", + # `store_statement_failed` (v1.10, §2f): a managed store errored or rejected a provisioner- + # ISSUED statement (CREATE/DROP/ALTER DATABASE, sentinel or canary probe) after passing + # readiness — the harness's own statements, so a deterministic failure here is a harness/ + # engine fault, `infrastructure` domain (retryable), never `seed_failed` (that code is + # reserved for the customer's own migration/seed content). + "store_statement_failed", +}) +# `ProcessRuntimeError` also raises codes that are deliberately INTERNAL-only — each marks a +# precondition `preflight_bundle` should already have made impossible (a placeholder token or a +# missing credential preflight itself should have caught), so by the module's own docstring these +# "never cross the outbound seam directly" and have no §2f entry to begin with. Excluded from +# CONTAINMENT, not from extraction — a genuinely new internal code still surfaces in the raised +# set for a human to classify, since only these documented names are exempted. +_INTERNAL_ONLY_RUNTIME_CODES = frozenset({ + "internal_unknown_placeholder", "internal_missing_credentials", + # Phase 6: marks a bundle-shape/state invariant an earlier layer (the model layer, or this + # module's own baseline-freeze-before-clone ordering) should already guarantee — e.g. + # `reset()` called before `provision()`, or a store's backing service turning out not to be a + # `ManagedProcess` despite `bundle_v2`'s `store_service_not_managed` check. A bug to fix here, + # never a bundle defect the outbound seam needs a name for — same status as the other two. + "internal_invariant_violated", +}) + + +def test_every_section_2f_code_process_runtime_raises_is_in_the_closed_table() -> None: + raised = _raised_codes_at_index( + Path(inspect.getfile(process_runtime_module)).read_text(encoding="utf-8"), + "ProcessRuntimeError", index=1, + ) + # `process_name_invalid` (F3's defense-in-depth path-containment check, `_ensure_within`) is + # not a NEW §2f code — it deliberately reuses the EXISTING §2e model-layer code as a backstop + # for when the model layer is bypassed, so `_SECTION_2E_CODES` is a legitimate source too, not + # just §2f's own six entries. + unlisted = raised - _SECTION_2F_CODES - _SECTION_2E_CODES - _INTERNAL_ONLY_RUNTIME_CODES + assert not unlisted, ( + f"raised but not in §2f's table (nor §2e's, nor a documented internal-only code): " + f"{sorted(unlisted)}" + ) + + +def test_the_section_2f_extraction_itself_finds_a_nonempty_set() -> None: + raised = _raised_codes_at_index( + Path(inspect.getfile(process_runtime_module)).read_text(encoding="utf-8"), + "ProcessRuntimeError", index=1, + ) + assert "build_failed" in raised + assert "spawn_failed" in raised + assert "source_tree_unavailable" in raised + assert "unsupported_capability_protocol" in raised + +def test_parallel_static_agent_name_is_refused(tmp_path: Path) -> None: + def overrides(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["LIVEKIT_AGENT_NAME"] = "static-agent" + return body + + manifest = _build_bundle(tmp_path, body_overrides=overrides) + with pytest.raises(PreflightError, match="agent_name_not_world_unique"): + preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + + +def test_parallel_world_indexed_agent_name_passes_the_guard(tmp_path: Path) -> None: + manifest = _build_bundle(tmp_path) + preflight_bundle(tmp_path, manifest, parallelism=2, secret_refs=TARGET_PROVIDER_REFS) + + +def test_single_world_static_agent_name_is_allowed(tmp_path: Path) -> None: + def overrides(body: dict[str, Any]) -> dict[str, Any]: + body["processes"][1]["environment"]["LIVEKIT_AGENT_NAME"] = "static-agent" + return body + + manifest = _build_bundle(tmp_path, body_overrides=overrides) + preflight_bundle(tmp_path, manifest, parallelism=1, secret_refs=TARGET_PROVIDER_REFS) + diff --git a/tests/harness/test_process_runtime.py b/tests/harness/test_process_runtime.py new file mode 100644 index 00000000..ebee7f9a --- /dev/null +++ b/tests/harness/test_process_runtime.py @@ -0,0 +1,4661 @@ +"""The execution half of the provisioner (`process_runtime.py`), per `hosted-execution-seams.md` +v1.12 §2b/§3/§4/§5. Manifests here are built directly through `EnvironmentBundleV2.model_validate` — +no digest sealing, no on-disk bundle — since every rule under test needs nothing but the parsed +model's own field values plus the job-supplied inputs (instances, secrets, credentials). Digest +and file-content verification are `test_process_preflight.py`'s job, not this module's. + +No Docker, no real postgres/redis/rabbitmq anywhere: managed-engine spawn is exercised only +through fakes (`ProcessRunner`, `sync_run`), since §0 assumes those binaries are on the snapshot's +PATH and this test lane must not require that. Real subprocess spawn IS exercised, through tiny +`python3 -c ...` stand-in scripts — proving `default_process_runner` genuinely spawns, captures +output, and reports liveness, without any engine-specific plumbing. + +P5's fix pass (F1/F2/F3/F7) made real filesystem operations security-relevant, so those are +exercised directly against real symlinks/tmp dirs where a fake would hide the exact class of bug +being fixed. `os.chown`/`Popen(user=...)` to a FOREIGN uid needs root, which this lane does not +have and must not require — those paths are verified STRUCTURALLY: a fake `chown`/`user_resolver` +is injected and the call it WOULD make is asserted, rather than performing the real privileged +syscall. + +Phase 6 (seed/baseline/worlds/reset/conformance/provision/close) adds `SqlSpy` below: a fake +`SqlRunner` that records every `(dbname, statement)` call in order and simulates just enough +postgres semantics (CREATE/DROP/ALTER DATABASE, `to_regclass`, row counts) for the baseline-freeze +and reset state machines to run against — no real postgres anywhere, same rule as everything +above. `asyncio.run` drives every `async def` seam (`ProcessRuntimeProvider.provision`/`reset`/ +`close`) directly; there is no event loop fixture here, matching this repo's existing async test +style elsewhere in the harness suite. +""" + +from __future__ import annotations + +import asyncio +import json +import subprocess +import sys +import time +import types +from dataclasses import replace as dc_replace +from pathlib import Path +from typing import Any, Callable + +import pytest + +from fi.alk.harness.bundle import CapabilityProtocol +from fi.alk.harness.bundle_v2 import ( + BUNDLE_V2_SCHEMA_VERSION, + CapabilityV2, + EnvironmentBundleV2, + ProcessUser, + SourceProcess, +) +from fi.alk.harness import process_runtime as pr + +# --- shared manifest builder ----------------------------------------------------------------- +# +# One postgres store (job-shared, `template_database`), one `tools-api` http-capability source +# process, one `agent` control-service source process claiming the job's only `target_provider` +# secret — the same shape `test_process_preflight.py` uses, so port/ordinal numbers a reader +# already knows from that file carry over here. + + +def _body(mutate: Callable[[dict[str, Any]], dict[str, Any]] | None = None) -> dict[str, Any]: + body: dict[str, Any] = { + "schema_version": BUNDLE_V2_SCHEMA_VERSION, + "name": "demo", + "digest": "sha256:" + "0" * 64, + "runtime": {"kind": "process", "control_service": "agent", "evidence_seam": "http_tool"}, + "processes": [ + { + "name": "postgres", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": [], + }, + { + "name": "tools-api", "kind": "source", "working_directory": "services/tools-api", + "build_commands": [["npm", "ci"]], "run_command": ["node", "server.js"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "PORT": "{{PORT_tools-api}}", + "TMPDIR": "{{WORLD_DIR}}", + }, + "secret_purposes": [], "user": "svc-tools", "depends_on": ["postgres"], + }, + { + "name": "agent", "kind": "source", "working_directory": ".", + "build_commands": [["pip", "install", "-r", "requirements.txt"]], + "run_command": ["python3", "agent.py"], + "environment": { + "DATABASE_URL": "{{DATABASE_URL}}", "TOOLS_API_URL": "{{TOOLS_API_URL}}", + "NAME": "agent-w{{WORLD_INDEX}}", + }, + "secret_purposes": ["target_provider"], "user": "svc-agent", + "depends_on": ["postgres", "tools-api"], + }, + ], + "capabilities": { + "database": { + "protocol": "postgres", "service": "postgres", "configuration_name": "DATABASE_URL", + }, + "tools": { + "protocol": "http", "service": "tools-api", "configuration_name": "TOOLS_API_URL", + }, + }, + "readiness": [ + { + "capability": "tools", "path": "/health", "timeout_seconds": 5, + "interval_seconds": 0.1, + }, + ], + "seed": { + "stores": [ + { + "capability": "database", "migrations": [], "seed_files": [], + "baseline": { + "strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64 + }, + "sentinel": {"query": "SELECT 1", "expected": "1"}, + } + ] + }, + "provenance": { + "source_kind": "repository", "repository": "org/repo", "source_digest": "c" * 64 + }, + "metadata": {}, + } + return mutate(body) if mutate is not None else body + + +def _manifest( + mutate: Callable[[dict[str, Any]], dict[str, Any]] | None = None, +) -> EnvironmentBundleV2: + return EnvironmentBundleV2.model_validate(_body(mutate)) + + +def _source_process(**overrides: Any) -> SourceProcess: + fields: dict[str, Any] = { + "name": "svc", "kind": "source", "working_directory": ".", "build_commands": [], + "run_command": ["python3", "-c", "pass"], "environment": {}, "secret_purposes": [], + "user": ProcessUser.SVC_TOOLS, "depends_on": [], + } + fields.update(overrides) + return SourceProcess(**fields) + + +def _solo_port_plan(process_name: str, *, ordinal: int = 0) -> pr.PortPlan: + """A single-process `PortPlan` for tests that spawn a `_source_process()` standalone, off the + shared `_manifest()` topology — `spawn_source_process` always looks its own process up in the + plan it is given, so the plan must actually know that process's name.""" + return pr.PortPlan( + ordinals={process_name: ordinal}, job_shared=frozenset(), fixed_ports={}, + effective_instances=1, degraded_reason=None, + ) + + +class FakeHandle: + """A `SpawnedProcess` fake: no real subprocess, just a captured-output buffer. `wait()` + reports the fake as exited the instant `terminate()`/`interrupt()`/`kill()` has been called — + no real process to actually wait on — which is enough for `_terminate_and_wait` (M7) to take + its happy path without ever escalating to `kill()` in a test. `StubbornHandle` (below, + N13-specific) is the deliberately-does-not-cooperate counterpart used to reach the kill branch.""" + + def __init__(self, output: str = "") -> None: + self._output = output + self.terminated = False + self.interrupted = False + self.killed = False + + def is_running(self) -> bool: + return not self.terminated + + def captured_output(self) -> str: + return self._output + + def terminate(self) -> None: + self.terminated = True + + def interrupt(self) -> None: + # N12, p6-review-r2: same instant-exit fake shape as `terminate()` — this fake exists to + # prove WHICH signal a caller preferred (`self.interrupted`), not to model postgres's own + # shutdown semantics. + self.interrupted = True + self.terminated = True + + def wait(self, timeout: float) -> bool: + return self.terminated + + def kill(self) -> None: + self.killed = True + self.terminated = True + + +class _FakePasswd: + """Stands in for `pwd.struct_passwd` — only `pw_uid`/`pw_gid` are ever read by this module.""" + + def __init__(self, uid: int, gid: int) -> None: + self.pw_uid = uid + self.pw_gid = gid + + +def _fake_user_resolver(known: dict[str, tuple[int, int]]) -> Callable[[str], _FakePasswd | None]: + def resolver(username: str) -> _FakePasswd | None: + if username in known: + uid, gid = known[username] + return _FakePasswd(uid, gid) + return None + + return resolver + + +def _reap(handle: Any, *, attempts: int = 50, interval: float = 0.05) -> None: + """Waits for a real spawned subprocess to exit. `pytest.fail`s loudly on exhaustion rather + than falling through to an assertion that would blame the wrong thing — a hung reap here means + the child never exited, not that its output was wrong.""" + for _ in range(attempts): + if not handle.is_running(): + return + time.sleep(interval) + else: + pytest.fail(f"subprocess did not exit within {attempts * interval}s") + + +# --- §2b port allocation ------------------------------------------------------------------------ + + +def test_ordinals_follow_the_authored_processes_array_order() -> None: + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.ordinals == {"postgres": 0, "tools-api": 1, "agent": 2} + + +def test_a_template_database_managed_engine_is_job_shared() -> None: + """§2b: `template_database` -> once per job; `14000 + ordinal`, the same in every world.""" + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.is_job_shared("postgres") + assert plan.port_for("postgres", 0) == 14000 + assert plan.port_for("postgres", 2) == 14000 + + +def test_source_processes_use_the_per_world_stride_formula() -> None: + """§2b: `15000 + 100*world_index + ordinal`.""" + plan = pr.plan_ports(_manifest(), instances=3) + assert plan.port_for("tools-api", 0) == 15001 + assert plan.port_for("tools-api", 1) == 15101 + assert plan.port_for("agent", 2) == 15202 + + +def test_a_datadir_copy_managed_engine_is_per_world_not_job_shared() -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [ + {**body["seed"]["stores"][0], "baseline": { + "strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64 + }}, + ]}} + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("postgres") + assert plan.port_for("postgres", 0) == 15000 + assert plan.port_for("postgres", 1) == 15100 + + +def test_a_managed_engine_with_no_seed_entry_at_all_defaults_to_per_world() -> None: + """§2b: "per-world is the only safe default" when a managed engine has no `seed.stores` + entry — a shared engine one world reset could otherwise corrupt for the others. + + T4, p5-round1-review: a REDIS capability with no store entry (§2c: implicitly `empty`, safe + with no `seed` block at all) rather than the old postgres one — a postgres capability with no + store entry is preflight-*invalid* (`seed_missing`), so the old fixture proved this branch on + a manifest that could never reach the provisioner for real. `postgres`'s own `seed.stores` + entry (`template_database`) is left in place, unrelated to the branch under test. + """ + manifest = _manifest( + lambda body: { + **body, + "processes": [*body["processes"], { + "name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": [], + }], + "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": None}, + }, + } + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("cache") + assert plan.port_for("cache", 0) != plan.port_for("cache", 1) + + +def test_the_empty_baseline_strategy_managed_engine_is_per_world() -> None: + """T5, p5-round1-review: `empty` is one of §2b's four instancing branches + (`template_database` job-shared; `datadir_copy`, `empty`, and no-seed-entry-at-all all + per-world) and had no direct test — only `datadir_copy` and the no-entry case were covered. + Redis is the only catalog engine whose strategies include `empty` + (`bundle_v2._ENGINE_STRATEGIES`).""" + manifest = _manifest( + lambda body: { + **body, + "processes": [*body["processes"], { + "name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": [], + }], + "capabilities": { + **body["capabilities"], + "cache": { + "protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL", + }, + }, + "seed": {"stores": [ + body["seed"]["stores"][0], + { + "capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "_seeded", "expected": "1"}, + }, + ]}, + } + ) + plan = pr.plan_ports(manifest, instances=2) + assert not plan.is_job_shared("cache") + assert plan.port_for("cache", 0) == 15003 # ordinal 3 (postgres,tools-api,agent,cache), world 0 + assert plan.port_for("cache", 1) == 15103 # ordinal 3, world 1 + + +def test_fixed_port_is_honored_exactly_and_forces_effective_instances_to_one() -> None: + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], + {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + plan = pr.plan_ports(manifest, instances=5) + assert plan.effective_instances == 1 + assert plan.degraded_reason == "fixed_port" + # Honored exactly, in every world index — there is only ever world 0 once degraded, but the + # allocator itself must not silently fall back to the formula for any index it is asked for. + assert plan.port_for("tools-api", 0) == 8081 + assert plan.port_for("tools-api", 3) == 8081 + + +def test_no_fixed_port_leaves_parallelism_undegraded() -> None: + plan = pr.plan_ports(_manifest(), instances=5) + assert plan.effective_instances == 5 + assert plan.degraded_reason is None + + +def test_port_band_boundary_ordinal_99_world_7_equals_15799() -> None: + """T6, p5-round1-review: the tiling is exact with zero slack (ordinal in [0,99], world in + [0,7], per §2e item 7's caps) — nothing exercised the actual boundary before.""" + plan = pr.PortPlan( + ordinals={"last": 99}, job_shared=frozenset(), fixed_ports={}, effective_instances=8, + degraded_reason=None, + ) + assert plan.port_for("last", 7) == 15799 + + +def test_job_shared_and_per_world_port_bands_are_disjoint() -> None: + """T6: 900 ports of slack between [14000,14099] (job-shared) and [15000,15799] (per-world) — + no ordinal/world combination within the contract's own caps can alias across bands.""" + plan = pr.PortPlan( + ordinals={"shared": 99, "world": 99}, job_shared=frozenset({"shared"}), + fixed_ports={}, effective_instances=8, degraded_reason=None, + ) + job_shared_ports = {plan.port_for("shared", w) for w in range(8)} + per_world_ports = {plan.port_for("world", w) for w in range(8)} + assert job_shared_ports == {14099} # ordinal 99, the same port in every world + assert per_world_ports == {15099, 15199, 15299, 15399, 15499, 15599, 15699, 15799} + assert job_shared_ports.isdisjoint(per_world_ports) + + +# --- §2b placeholder renderer -------------------------------------------------------------------- + + +def test_configuration_name_placeholder_renders_the_capabilitys_address(tmp_path: Path) -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + endpoints = pr.build_endpoints(manifest, world_index=1, port_plan=plan, credentials=credentials) + addresses = pr.configuration_addresses_from_endpoints(endpoints) + agent = manifest.processes[2] + rendered = pr.render_environment( + agent, world_index=1, world_dir=tmp_path / "worlds" / "w1" / "agent", + port_plan=plan, configuration_addresses=addresses, + ) + assert rendered["DATABASE_URL"] == "postgresql://harness:PW@localhost:14000/w1" + assert rendered["TOOLS_API_URL"] == "http://localhost:15101" + assert rendered["NAME"] == "agent-w1" + + +def test_world_dir_db_name_host_and_port_placeholders(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], + { + **body["processes"][1], + "environment": { + **body["processes"][1]["environment"], + "SCRATCH": "{{WORLD_DIR}}", + "DB": "{{DB_NAME}}", + "PEER": "{{HOST_postgres}}:{{PORT_postgres}}", + }, + }, + body["processes"][2], + ], + } + ) + plan = pr.plan_ports(manifest, instances=1) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + endpoints = pr.build_endpoints(manifest, world_index=0, port_plan=plan, credentials=credentials) + addresses = pr.configuration_addresses_from_endpoints(endpoints) + world_dir = tmp_path / "worlds" / "w0" / "tools-api" + tools_api = manifest.processes[1] + rendered = pr.render_environment( + tools_api, world_index=0, world_dir=world_dir, port_plan=plan, + configuration_addresses=addresses, + ) + assert rendered["SCRATCH"] == str(world_dir) + assert rendered["DB"] == "w0" + assert rendered["PEER"] == "localhost:14000" + + +def test_world_index_placeholder_renders_the_bare_integer() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + rendered = pr.render_template( + "world-{{WORLD_INDEX}}", process_name="agent", world_index=3, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert rendered == "world-3" + + +def test_an_unresolvable_token_is_an_internal_error_not_a_preflight_code() -> None: + """`preflight_bundle` already validated every token against the closed vocabulary before this + ever runs — a token this function cannot resolve is a bug, not a bundle defect, so it is NOT + one of `PreflightError`'s §2e codes.""" + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_template( + "{{NOT_A_REAL_TOKEN}}", process_name="agent", world_index=0, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert excinfo.value.code == "internal_unknown_placeholder" + assert excinfo.value.stage == "render" + + +def test_a_port_placeholder_naming_an_unknown_process_is_also_an_internal_error() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_template( + "{{PORT_ghost}}", process_name="agent", world_index=0, world_dir=Path("/x"), + port_plan=plan, configuration_addresses={}, + ) + assert excinfo.value.code == "internal_unknown_placeholder" + + +def test_build_environment_is_merged_raw_and_never_templated(tmp_path: Path) -> None: + """§2b: `build_environment` "takes NO placeholders at all" — proven end to end through + `spawn_source_process`'s env, since `render_environment`/`render_template` are never even + called on it.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process(build_environment={"TMPDIR": "{{WORLD_DIR}}"}) + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["env"] = env + return FakeHandle() + + plan = _solo_port_plan("svc") + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert captured["env"]["TMPDIR"] == "{{WORLD_DIR}}" + + +def test_render_capability_address_raises_for_an_unsupported_protocol() -> None: + """F10, p5-round1-review: a `mongodb`/`s3`/`kafka`/... capability has no defined address + shape at this seam (§3 names exactly two worked examples) — this used to silently render a + bare `://host:port`, which is not a working address for any of them.""" + capability = CapabilityV2( + protocol="mongodb", service="svc", configuration_name="MONGO_URL" + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_capability_address(capability, port=15005, world_index=0, credentials=None) + assert excinfo.value.code == "unsupported_capability_protocol" + + +def test_render_capability_address_raises_for_missing_postgres_credentials() -> None: + """S11, p5-round1-review: a typed raise for this precondition, not a bare `assert` (stripped + under `python -O`) — the same defect class as P4's N8(2).""" + capability = CapabilityV2( + protocol="postgres", service="postgres", configuration_name="DATABASE_URL" + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.render_capability_address(capability, port=14000, world_index=0, credentials=None) + assert excinfo.value.code == "internal_missing_credentials" + + +# --- secrets: purpose filtering (F13) ----------------------------------------------------------- + + +def test_select_process_secrets_matches_only_the_claimed_purpose() -> None: + process = _source_process(secret_purposes=["target_provider"]) + selected = pr.select_process_secrets( + process, + secret_values={"A": "1", "B": "2"}, + secret_purposes={"A": "target_provider", "B": "source_checkout"}, + ) + assert selected == {"A": "1"} + + +def test_select_process_secrets_hard_excludes_source_checkout_even_if_claimed() -> None: + """F13, p5-round1-review: §1 states `source_checkout` is gateway-only and never uploaded to + the guest — preflight's `secret_unclaimed`/`secret_missing` pair is scoped to + `target_provider` only, so a process CAN legally claim `source_checkout` too. This function + must not depend on the gateway alone to keep that promise.""" + process = _source_process(secret_purposes=["target_provider", "source_checkout"]) + selected = pr.select_process_secrets( + process, + secret_values={"A": "target-provider-value", "B": "checkout-value-must-not-move"}, + secret_purposes={"A": "target_provider", "B": "source_checkout"}, + ) + assert selected == {"A": "target-provider-value"} + + +# --- env construction (F12/F14) ----------------------------------------------------------------- + + +def test_allowlisted_ambient_env_keeps_only_the_fixed_set() -> None: + source = { + "PATH": "/bin", "HOME": "/root", "LANG": "C", "TZ": "UTC", "TMPDIR": "/tmp", + "LC_ALL": "C", "SECRET_TOKEN": "leak-me", "AWS_SECRET_ACCESS_KEY": "leak-me-too", + } + env = pr._allowlisted_ambient_env(source) + assert env == { + "PATH": "/bin", "HOME": "/root", "LANG": "C", "TZ": "UTC", "TMPDIR": "/tmp", "LC_ALL": "C", + } + + +def test_spawn_source_process_env_does_not_inherit_an_arbitrary_ambient_var( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """F12, p5-round1-review: §2b enumerates exactly what a process receives — the ambient + `svc-control` environment (a future bearer token, a `FUTUREAGI_*` marker) is not on that + list.""" + monkeypatch.setenv("FUTUREAGI_PROVISIONER_MARKER", "leak-me") + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["env"] = env + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert "FUTUREAGI_PROVISIONER_MARKER" not in captured["env"] + + +def test_base_process_env_path_prepend_has_no_empty_trailing_element(tmp_path: Path) -> None: + """F14, p5-round1-review: an unset/empty inherited `PATH` used to leave a trailing `:`, and + POSIX `execvp` reads an empty PATH element as "current directory" — cwd for build/run is the + customer's own tree, so a repo shipping an executable literally named `ls`/`git`/`sh` could + shadow the real one for any bare-name argv[0].""" + env = pr._base_process_env(tmp_path / "build" / "svc", base={}) + assert "" not in env["PATH"].split(":") + + +# --- §2b copy-based build trees ------------------------------------------------------------------- + + +def test_build_process_tree_copies_the_working_directory(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + (tmp_path / "source" / "svc" / "main.py").write_text("print(1)\n") + (tmp_path / "source" / "svc" / "sub").mkdir() + (tmp_path / "source" / "svc" / "sub" / "helper.py").write_text("print(2)\n") + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert build_dir == tmp_path / "build" / "svc" + assert (build_dir / "main.py").read_text() == "print(1)\n" + assert (build_dir / "sub" / "helper.py").read_text() == "print(2)\n" + + +def test_build_commands_get_the_venv_and_node_modules_path_prepend(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + calls: list[list[str]] = [] + + def fake_run(step, *, cwd, env, **kwargs): + calls.append(env["PATH"].split(":")[:2]) + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["npm", "ci"], ["npm", "build"]] + ) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + expected = [str(build_dir / ".venv" / "bin"), str(build_dir / "node_modules" / ".bin")] + assert len(calls) == 2 # every step, not just the first + assert calls[0] == expected + assert calls[1] == expected + + +def test_build_environment_is_merged_into_the_build_step_env(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + captured: dict[str, Any] = {} + + def fake_run(step, *, cwd, env, **kwargs): + captured["FOO"] = env.get("FOO") + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["true"]], build_environment={"FOO": "bar"} + ) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert captured["FOO"] == "bar" + + +def test_build_runs_every_step_once_per_call_in_order(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + order: list[list[str]] = [] + + def fake_run(step, *, cwd, env, **kwargs): + order.append(step) + return subprocess.CompletedProcess(step, 0) + + process = _source_process( + working_directory="svc", build_commands=[["step-a"], ["step-b"], ["step-c"]] + ) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert order == [["step-a"], ["step-b"], ["step-c"]] + + +def test_build_process_trees_builds_each_source_process_exactly_once(tmp_path: Path) -> None: + """No world loop exists inside `build_process_trees` at all — structural proof of "once per + job," not "once per world.\"""" + (tmp_path / "source" / "services" / "tools-api").mkdir(parents=True) + (tmp_path / "source" / ".").mkdir(exist_ok=True) + manifest = _manifest() + calls: list[str] = [] + + def fake_run(step, *, cwd, env, **kwargs): + calls.append(str(cwd)) + return subprocess.CompletedProcess(step, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=pr.plan_ports(manifest, instances=4), credentials={}, + secret_values={}, secret_purposes={}, sync_run=fake_run, + ) + build_dirs = pr.build_process_trees(manifest, source_root=tmp_path / "source", context=context) + assert set(build_dirs) == {"tools-api", "agent"} # every `source` process, no managed ones + # tools-api has 1 build step, agent has 1 build step — exactly 2 calls total, never per-world. + assert len(calls) == 2 + + +def test_a_nonzero_build_step_is_build_failed(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + return subprocess.CompletedProcess(step, 1, stdout="", stderr="npm ERR! boom") + + process = _source_process(working_directory="svc", build_commands=[["npm", "ci"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.stage == "build" + assert excinfo.value.code == "build_failed" + assert "boom" in str(excinfo.value) + + +def test_a_missing_interpreter_is_runtime_unsupported(tmp_path: Path) -> None: + """§0 (v1.8): "a repo needing an interpreter the snapshot lacks fails at BUILD time... the + build step's failure is reported `runtime_unsupported`.\"""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise FileNotFoundError(f"no such file: {step[0]!r}") + + process = _source_process( + working_directory="svc", build_commands=[["python3.13", "-m", "venv", ".venv"]] + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.stage == "build" + assert excinfo.value.code == "runtime_unsupported" + + +def test_a_missing_non_interpreter_build_tool_is_build_failed_not_runtime_unsupported( + tmp_path: Path, +) -> None: + """The missing-interpreter detection is scoped to python*/node* argv[0] patterns (§0 only + promises those two families) — a missing custom build tool is a `build_failed`, not a + `runtime_unsupported`, since the snapshot never promised it in the first place.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise FileNotFoundError(f"no such file: {step[0]!r}") + + process = _source_process(working_directory="svc", build_commands=[["some-custom-tool", "x"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run + ) + assert excinfo.value.code == "build_failed" + + +def test_a_build_step_that_times_out_is_build_failed(tmp_path: Path) -> None: + """F15, p5-round1-review: an install step wedged on a private registry with no DNS answer + used to block the provisioner forever.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + + def fake_run(step, *, cwd, env, **kwargs): + raise subprocess.TimeoutExpired(cmd=step, timeout=kwargs.get("timeout", 0)) + + process = _source_process(working_directory="svc", build_commands=[["npm", "ci"]]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run, + build_step_timeout_seconds=5, + ) + assert excinfo.value.code == "build_failed" + + +# --- F5: typed copy-phase failures ---------------------------------------------------------------- + + +def test_build_process_tree_missing_working_directory_is_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source").mkdir() + process = _source_process(working_directory="does-not-exist", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + assert excinfo.value.stage == "build" + + +def test_build_process_tree_working_directory_naming_a_file_is_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source").mkdir() + (tmp_path / "source" / "svc").write_text("not a directory") + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + + +def test_build_process_tree_wraps_a_copy_failure_as_source_tree_unavailable( + tmp_path: Path, +) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + + def failing_copy(src: Path, dst: Path) -> None: + raise PermissionError("no") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + copy=failing_copy, + ) + assert excinfo.value.code == "source_tree_unavailable" + + +# --- F2 (BLOCKER): symlink safety, exercised against a real filesystem --------------------------- + + +def test_build_process_tree_rejects_a_symlink_escaping_the_source_root(tmp_path: Path) -> None: + """F2, p5-round1-review — BLOCKER. A repo can ship a symlink pointing outside `/work/source` + (e.g. at `/run/futureagi/capabilities.json`); `copytree`'s default (`symlinks=False`) used to + dereference it and copy the TARGET's bytes into the customer's own build tree, read as + svc-control. Verified against a REAL symlink and a REAL filesystem, not a fake copier — this + is exactly the class of bug a fake would hide.""" + outside_secret = tmp_path / "outside_secret.txt" + outside_secret.write_text("TOP SECRET") + svc_dir = tmp_path / "source" / "svc" + svc_dir.mkdir(parents=True) + (svc_dir / "evil_link").symlink_to(outside_secret) + + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert excinfo.value.code == "source_tree_unavailable" + # Rejected BEFORE the copy, not after — nothing was ever materialized. + assert not (tmp_path / "build" / "svc").exists() + + +def test_build_process_tree_preserves_a_within_tree_symlink_as_a_symlink(tmp_path: Path) -> None: + """A link that stays inside the tree is legitimate and must keep working — `symlinks=True` + copies it AS a link, it does not forbid it.""" + svc_dir = tmp_path / "source" / "svc" + svc_dir.mkdir(parents=True) + (svc_dir / "real.txt").write_text("hello") + (svc_dir / "link.txt").symlink_to(svc_dir / "real.txt") + + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build" + ) + assert (build_dir / "link.txt").is_symlink() + assert (build_dir / "link.txt").read_text() == "hello" + + +def test_build_process_tree_rejects_a_symlinked_working_directory_path_component( + tmp_path: Path, +) -> None: + """F2: the resolve-then-`is_relative_to` check catches a symlinked PATH COMPONENT too, not + just a symlinked leaf file — `working_directory: "services/tools-api"` passes the model + layer's `_safe_relative` (no `..`, not absolute) even when `/work/source/services` is ITSELF + a symlink pointing outside the checkout.""" + source_root = tmp_path / "source" + source_root.mkdir() + (source_root / "services").symlink_to(tmp_path) # escapes source_root entirely + + process = _source_process(working_directory="services/tools-api", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree(process, source_root=source_root, build_root=tmp_path / "build") + assert excinfo.value.code == "source_tree_unavailable" + + +# --- F3 (BLOCKER): path-containment defense in depth ---------------------------------------------- + + +def test_build_tree_dir_rejects_a_name_that_escapes_the_work_directory(tmp_path: Path) -> None: + """F3, p5-round1-review — BLOCKER. Defense in depth, independent of the model-layer regex + (`bundle_v2.SourceProcess`/`ManagedProcess.name`'s pattern) — these helpers take a plain + `str`, so a caller bypassing model validation still cannot walk a name-derived directory + outside `work_directory`. `pathlib` makes an absolute name a one-field catastrophe otherwise: + `Path("/work/build") / "/etc"` IS `Path("/etc")`, which the old code then `rmtree`'d.""" + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_tree_dir(tmp_path, "/etc") + assert excinfo.value.code == "process_name_invalid" + + +def test_world_scratch_dir_rejects_a_traversal_name_that_truly_escapes(tmp_path: Path) -> None: + # `world_scratch_dir` prepends TWO fixed levels ("worlds", "w") before the name — 3 levels + # of ".." is needed to escape `work_directory` itself (2 would only cancel the two prepended + # levels and land back inside it, which is not an escape). + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.world_scratch_dir(tmp_path, 0, "../../../etc") + assert excinfo.value.code == "process_name_invalid" + + +def test_managed_engine_data_dir_rejects_a_traversal_name_in_both_branches(tmp_path: Path) -> None: + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.managed_engine_data_dir(tmp_path, "/etc", world_index=None) + assert excinfo.value.code == "process_name_invalid" + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.managed_engine_data_dir(tmp_path, "../../../etc", world_index=0) + assert excinfo.value.code == "process_name_invalid" + + +def test_legitimate_process_names_are_unaffected_by_the_containment_check(tmp_path: Path) -> None: + assert pr.build_tree_dir(tmp_path, "tools-api") == tmp_path / "build" / "tools-api" + assert pr.world_scratch_dir(tmp_path, 2, "agent") == tmp_path / "worlds" / "w2" / "agent" + assert pr.managed_engine_data_dir(tmp_path, "postgres", world_index=None) == ( + tmp_path / "managed" / "postgres" + ) + + +# --- F1 (BLOCKER): `user` honored — chown + privilege drop, verified structurally ----------------- + + +def test_build_process_tree_chowns_the_copied_tree_to_the_resolved_user(tmp_path: Path) -> None: + """F1, p5-round1-review — BLOCKER. The build tree (root AND every copied file, not just the + directory) is chowned to the process's declared `user` after copy. `os.chown` to a foreign uid + needs root, which this lane does not have and must not require — `chown` is faked here and the + calls it WOULD make are asserted; see the module docstring.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + (tmp_path / "source" / "svc" / "main.py").write_text("print(1)\n") + process = _source_process(working_directory="svc", build_commands=[]) + chowned: list[tuple[str, int, int]] = [] + + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + build_dir = tmp_path / "build" / "svc" + assert (str(build_dir), 1234, 5678) in chowned + assert (str(build_dir / "main.py"), 1234, 5678) in chowned + + +def test_build_commands_run_under_the_resolved_user(tmp_path: Path) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + captured: dict[str, Any] = {} + + def fake_run(step, *, cwd, env, **kwargs): + captured["user"] = kwargs.get("user") + captured["group"] = kwargs.get("group") + return subprocess.CompletedProcess(step, 0) + + process = _source_process(working_directory="svc", build_commands=[["true"]]) + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", run=fake_run, + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + # `chown` faked too — real `os.chown` to a fake uid needs root, which this lane does not + # have; the chown call itself is asserted separately (`test_build_process_tree_chowns_...`). + chown=lambda path, uid, gid: None, + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + + +def test_build_process_tree_falls_back_unprivileged_when_user_is_not_resolvable( + tmp_path: Path, +) -> None: + """The local test lane's own shape: no `svc-*` accounts on a dev box. + `require_declared_user=False` (the default) must NOT raise — it runs unprivileged instead.""" + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + build_dir = pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=lambda name: None, + ) + assert build_dir.is_dir() # did not raise + + +def test_build_process_tree_raises_when_user_is_required_but_not_resolvable( + tmp_path: Path, +) -> None: + (tmp_path / "source" / "svc").mkdir(parents=True) + process = _source_process(working_directory="svc", build_commands=[]) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.build_process_tree( + process, source_root=tmp_path / "source", build_root=tmp_path / "build", + user_resolver=lambda name: None, require_declared_user=True, + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_source_process_applies_the_resolved_user_to_the_runner(tmp_path: Path) -> None: + """Tests: assert the resolved user IS applied — the fake runner records `user=`.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + chowned: list[tuple[str, int, int]] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["user"] = user + captured["group"] = group + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=_fake_user_resolver({"svc-tools": (1234, 5678)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + # `{{WORLD_DIR}}` is chowned too — otherwise a process running as anyone but svc-control could + # not write into its own per-world scratch directory at all. + assert (str(world_dir), 1234, 5678) in chowned + + +def test_spawn_source_process_falls_back_unprivileged_when_user_is_not_resolvable( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["user"] = user + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=lambda name: None, + ) + assert captured["user"] is None + + +def test_spawn_source_process_raises_when_user_is_required_but_not_resolvable( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + user_resolver=lambda name: None, require_declared_user=True, + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_managed_process_chowns_data_dir_to_the_resolved_user(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + chowned: list[tuple[str, int, int]] = [] + + def fake_sync_run(argv, **kwargs): + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert (str(data_dir), 2222, 3333) in chowned + # The pwfile is chowned to the same user too — `initdb` runs as that user (below) and must be + # able to read its own 0600 file. + assert any(uid == 2222 and gid == 3333 and path.endswith(".pwfile") for path, uid, gid in chowned) + + +def test_spawn_managed_process_chowns_a_pre_populated_data_dir_recursively(tmp_path: Path) -> None: + """B6, p6-review-r1: `data_dir` is empty on the very FIRST spawn (`initdb` itself creates + everything under it as the already-correct user), but `freeze_baseline`'s `datadir_copy` + snapshot and `_seal_world_store`'s restore both populate it via a COPY that runs as the + provisioner — every file underneath is provisioner-owned, and postgres refuses to start + unless the data directory AND ITS CONTENTS are owned by the effective user. Simulates that by + pre-populating nested content before the spawn call, same as a restored `datadir_copy` world + would look like at this point, and asserts every path underneath was chowned, not only the + top-level directory a single non-recursive `chown` call would have caught.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + (data_dir / "base" / "1").mkdir(parents=True) + (data_dir / "PG_VERSION").write_text("16\n") # already looks bootstrapped; initdb is skipped. + (data_dir / "base" / "1" / "1234").write_text("data") + chowned: list[tuple[str, int, int]] = [] + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + chowned_paths = {path for path, _, _ in chowned} + assert str(data_dir) in chowned_paths + assert str(data_dir / "base") in chowned_paths + assert str(data_dir / "base" / "1") in chowned_paths + assert str(data_dir / "base" / "1" / "1234") in chowned_paths + assert all(uid == 2222 and gid == 3333 for _, uid, gid in chowned) + assert (data_dir.stat().st_mode & 0o777) == 0o700 + + +def test_default_process_runner_forwards_user_and_group_to_popen( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """F1: `subprocess.Popen(user=, group=)` requires root/CAP_SETUID to actually switch identity + — this lane does not have it and must not require it. Verified structurally: `Popen` itself is + faked, and the kwargs it would have been called with are asserted.""" + captured: dict[str, Any] = {} + + class FakePopen: + def __init__(self, *args: Any, **kwargs: Any) -> None: + captured.update(kwargs) + + def poll(self) -> None: + return None + + monkeypatch.setattr(pr.subprocess, "Popen", FakePopen) + chowned: list[tuple[str, int, int]] = [] + pr.default_process_runner( + ["true"], cwd=tmp_path, env={}, log_path=tmp_path / "x.log", user=1234, group=5678, + chown=lambda path, uid, gid: chowned.append((str(path), uid, gid)), + ) + assert captured["user"] == 1234 + assert captured["group"] == 5678 + # The log file the harness creates (before the child's privilege drop) is chowned to match — + # otherwise nothing running as the child's own user could ever open it fresh afterward. + assert chowned == [(str(tmp_path / "x.log"), 1234, 5678)] + + +def test_default_process_runner_does_not_chown_the_log_when_no_user_is_given( + tmp_path: Path, +) -> None: + chowned: list[Any] = [] + handle = pr.default_process_runner( + ["python3", "-c", "pass"], cwd=tmp_path, env={"PATH": "/usr/bin:/bin"}, + log_path=tmp_path / "logs" / "proc.log", + chown=lambda *args: chowned.append(args), + ) + _reap(handle) + assert chowned == [] + + +# --- process spawn: real subprocesses via `python3 -c`, secret injection ------------------------ + + +def test_default_process_runner_spawns_a_real_subprocess_and_captures_output( + tmp_path: Path, +) -> None: + log_path = tmp_path / "logs" / "proc.log" + handle = pr.default_process_runner( + ["python3", "-c", "print('hello-from-child')"], cwd=tmp_path, env={"PATH": "/usr/bin:/bin"}, + log_path=log_path, + ) + _reap(handle) + assert "hello-from-child" in handle.captured_output() + + +def test_a_process_claiming_the_purpose_receives_its_secret(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "agent" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "agent" + script = ( + "import os,sys; sys.stdout.write('SECRET=' + os.environ.get('LIVEKIT_API_KEY', ''))" + ) + process = _source_process( + run_command=["python3", "-c", script], secret_purposes=["target_provider"] + ) + plan = _solo_port_plan("svc") + handle = pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={"LIVEKIT_API_KEY": "abc123", "OTHER": "zzz"}, + secret_purposes={"LIVEKIT_API_KEY": "target_provider", "OTHER": "source_checkout"}, + ) + _reap(handle.handle) + assert handle.handle.captured_output() == "SECRET=abc123" + + +def test_a_process_not_claiming_the_purpose_does_not_receive_the_secret(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + script = ( + "import os,sys; sys.stdout.write('SECRET=' + os.environ.get('LIVEKIT_API_KEY', ''))" + ) + process = _source_process(run_command=["python3", "-c", script], secret_purposes=[]) + plan = _solo_port_plan("svc") + handle = pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={"LIVEKIT_API_KEY": "abc123"}, + secret_purposes={"LIVEKIT_API_KEY": "target_provider"}, + ) + _reap(handle.handle) + assert handle.handle.captured_output() == "SECRET=" + + +def test_spawn_source_process_creates_the_per_world_scratch_directory(tmp_path: Path) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w2" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=2, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert world_dir.is_dir() + + +def test_spawn_source_process_cwd_is_the_build_tree_not_the_world_scratch_dir( + tmp_path: Path, +) -> None: + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + world_dir = tmp_path / "worlds" / "w0" / "svc" + process = _source_process() + plan = _solo_port_plan("svc") + captured: dict[str, Any] = {} + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured["cwd"] = cwd + return FakeHandle() + + pr.spawn_source_process( + process, build_dir=build_dir, world_dir=world_dir, world_index=0, port_plan=plan, + configuration_addresses={}, secret_values={}, secret_purposes={}, runner=fake_runner, + ) + assert captured["cwd"] == build_dir + + +def test_spawn_managed_process_requires_credentials_for_postgres(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_managed_process( + postgres, port=14000, data_dir=tmp_path / "pg", credentials=None, + runner=lambda *a, **k: FakeHandle(), + ) + assert excinfo.value.code == "spawn_failed" + + +def test_spawn_managed_process_bootstraps_postgres_once_via_sync_run(tmp_path: Path) -> None: + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + bootstrap_calls: list[list[str]] = [] + run_calls: list[list[str]] = [] + + def fake_sync_run(argv, **kwargs): + bootstrap_calls.append(argv) + (data_dir).mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + run_calls.append(argv) + return FakeHandle() + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, runner=fake_runner, + sync_run=fake_sync_run, + ) + assert len(bootstrap_calls) == 1 + assert bootstrap_calls[0][0] == "initdb" + assert run_calls[0][0] == "postgres" + # No pwfile left behind after bootstrap. + assert not any(p.name.endswith(".pwfile") for p in data_dir.parent.glob(".*")) + + # A second spawn against the same already-initialized data_dir must not bootstrap again. + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, runner=fake_runner, + sync_run=fake_sync_run, + ) + assert len(bootstrap_calls) == 1 + + +# --- F7 (MAJOR): the pwfile is created 0600 atomically, not write-then-chmod --------------------- + + +def test_spawn_managed_process_creates_the_pwfile_with_o_excl_and_0600_atomically( + tmp_path: Path, +) -> None: + """F7, p5-round1-review. `write_text` then `os.chmod` creates the file at `0666 & ~umask` + (typically 0644) and only narrows it afterward — a classic create-then-chmod TOCTOU that + leaves the generated superuser password world-readable for the window in between. + `os.open`'s own arguments are captured directly (the mode at CREATION, not observed after the + fact, which cannot distinguish "created narrow" from "created wide, narrowed a moment later").""" + import os as os_module + + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + data_dir = tmp_path / "pg" + captured: dict[str, Any] = {} + real_open = os_module.open + + def spy_open(path, flags, mode=0o777, *args, **kwargs): + if str(path).endswith(".pwfile"): + captured["flags"] = flags + captured["mode"] = mode + return real_open(path, flags, mode, *args, **kwargs) + + def fake_sync_run(argv, **kwargs): + data_dir.mkdir(parents=True, exist_ok=True) + (data_dir / "PG_VERSION").write_text("16\n") + return subprocess.CompletedProcess(argv, 0) + + pr.os.open = spy_open + try: + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=fake_sync_run, + ) + finally: + pr.os.open = real_open + + assert captured["mode"] == 0o600 + assert captured["flags"] & os_module.O_EXCL + assert captured["flags"] & os_module.O_CREAT + + +# --- §2b depends_on wait ----------------------------------------------------------------------- + + +def test_depends_on_with_neither_readiness_nor_started_check_returns_immediately() -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("postgres", FakeHandle(), 14000, None) + calls = {"n": 0} + + def fake_prober(**kwargs): + calls["n"] += 1 + return True + + pr.wait_for_dependency( + manifest, "postgres", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + ) + assert calls["n"] == 0 # no readiness declared for `database` in this manifest -> immediate. + + +def test_depends_on_waits_for_the_capabilitys_readiness_probe() -> None: + manifest = _manifest() # `tools` capability has a declared readiness probe. + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + seen: list[tuple[str, int]] = [] + calls = {"n": 0} + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + calls["n"] += 1 + seen.append((host, port)) + return calls["n"] >= 3 + + ticks = {"t": 0.0} + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert calls["n"] == 3 + assert seen[0] == ("localhost", 15001) + + +def test_wait_for_dependency_requires_every_readiness_probe_backing_the_process() -> None: + """F8, p5-round1-review: a process backing two capabilities is only "ready" once ALL of its + declared readiness probes pass — matching `healthy()`'s own all-probes semantics (§4 point 3). + A process used to be treated as ready by `depends_on` the moment the FIRST-listed probe + passed, then immediately reported unhealthy by `healthy()` the instant the second had not.""" + manifest = _manifest( + lambda body: { + **body, + "capabilities": { + **body["capabilities"], + "tools_admin": { + "protocol": "http", "service": "tools-api", "configuration_name": None, + }, + }, + "readiness": [ + *body["readiness"], + { + "capability": "tools_admin", "path": "/admin/health", "timeout_seconds": 5, + "interval_seconds": 0.1, + }, + ], + } + ) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + ready = {"/health": False, "/admin/health": False} + seen_paths: list[str | None] = [] + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen_paths.append(path) + return ready[path] + + ticks = {"t": 0.0} + + def sleep(seconds: float) -> None: + ticks["t"] += seconds + if ticks["t"] >= 0.2: + ready["/health"] = True + if ticks["t"] >= 0.4: + ready["/admin/health"] = True + + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, prober=fake_prober, + clock=lambda: ticks["t"], sleep=sleep, + ) + assert "/admin/health" in seen_paths + assert ready["/health"] and ready["/admin/health"] + + +def test_depends_on_readiness_probe_timeout_is_a_typed_error() -> None: + manifest = _manifest() + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 15001, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + prober=lambda **kwargs: False, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.stage == "depends_on" + assert excinfo.value.code == "depends_on_timeout" + + +def test_started_check_log_marker_variant_waits_for_the_marker() -> None: + """T2, p5-round1-review: the marker now APPEARS after N polls (a mutating fake, same idiom as + `test_depends_on_waits_for_the_capabilitys_readiness_probe`'s `calls['n'] >= 3`) rather than + being seeded into `captured_output()` up front — the old fixture could not distinguish "polls + until the marker appears" from "checks once and returns.\"""" + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], + "started_check": {"log_marker": "listening", "timeout_seconds": 5}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + handle = FakeHandle("booting...\n") + spawned = pr.SpawnedWorldProcess("tools-api", handle, 15001, 0) + calls = {"n": 0} + + def mutating_captured_output() -> str: + calls["n"] += 1 + if calls["n"] >= 3: + handle._output = "booting...\nlistening on :9\n" + return handle._output + + handle.captured_output = mutating_captured_output + ticks = {"t": 0.0} + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert calls["n"] >= 3 + + +def test_started_check_log_marker_timeout_is_a_typed_error() -> None: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], + "started_check": {"log_marker": "listening", "timeout_seconds": 1}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + handle = FakeHandle("booting...\n") # marker never appears + spawned = pr.SpawnedWorldProcess("tools-api", handle, 15001, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.code == "depends_on_timeout" + + +def test_started_check_port_variant_probes_the_dependencys_own_allocated_port() -> None: + """F4, p5-round1-review — MAJOR. §2b (v1.8): `started_check.port` only SELECTS the port-probe + variant — the dialed port is always the dependency's OWN allocated port (honoring + `fixed_port`), never a literal. Exercised against a REAL bound socket on the port + `port_plan.port_for` computes (via `fixed_port`, so the exact port is deterministic rather + than depending on the formula's value not colliding with something else already listening).""" + import socket as socket_module + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + port = server.getsockname()[1] + try: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], "fixed_port": port, + "started_check": {"port": True, "timeout_seconds": 5}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + assert plan.port_for("tools-api", 0) == port + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), port, 0) + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned + ) + finally: + server.close() + + +def test_started_check_port_variant_dials_the_world_specific_formula_port( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """F4: without `fixed_port`, the dialed port must be world-`world_index`'s OWN formula port — + proven by monkeypatching the underlying TCP probe and asserting the exact port dialed, which a + real-server-bind test cannot show as directly (it can only prove "some port worked").""" + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + {**body["processes"][1], "started_check": {"port": True, "timeout_seconds": 5}}, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=3) + expected_port = plan.port_for("tools-api", 2) + assert expected_port == 15201 # ordinal 1, world 2 — NOT the same as world 0's 15001. + dialed: list[tuple[str, int]] = [] + + def fake_tcp_probe(host: str, port: int, *, timeout: float = 0.75) -> bool: + dialed.append((host, port)) + return True + + monkeypatch.setattr(pr, "_tcp_probe", fake_tcp_probe) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), expected_port, 2) + pr.wait_for_dependency(manifest, "tools-api", world_index=2, port_plan=plan, spawned=spawned) + assert dialed == [("localhost", expected_port)] + + +def test_started_check_port_variant_timeout_when_nothing_listens() -> None: + manifest = _manifest( + lambda body: {**body, "readiness": [], "processes": [ + body["processes"][0], + { + **body["processes"][1], "fixed_port": 1, + "started_check": {"port": True, "timeout_seconds": 1}, + }, + body["processes"][2], + ]} + ) + plan = pr.plan_ports(manifest, instances=1) + spawned = pr.SpawnedWorldProcess("tools-api", FakeHandle(), 1, 0) + ticks = {"t": 0.0} + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.wait_for_dependency( + manifest, "tools-api", world_index=0, port_plan=plan, spawned=spawned, + clock=lambda: ticks["t"], sleep=lambda s: ticks.__setitem__("t", ticks["t"] + s), + ) + assert excinfo.value.code == "depends_on_timeout" + + +def test_spawn_world_spawns_in_dependency_order_and_waits_between(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) # skip real readiness waits + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + spawn_order: list[str] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawn_order.append(argv[0]) + return FakeHandle() + + def fake_sync_run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=plan, credentials=credentials, secret_values={}, + secret_purposes={}, runner=fake_runner, sync_run=fake_sync_run, + ) + result = pr.spawn_world(manifest, world_index=0, context=context) + assert set(result.handles) == {"postgres", "tools-api", "agent"} + # postgres has no dependency; tools-api depends on postgres; agent depends on both. + assert spawn_order.index("postgres") < spawn_order.index("node") # tools-api's run_command[0] + assert spawn_order.index("node") < spawn_order.index("python3") # agent's run_command[0] + + +def test_spawn_world_reuses_job_shared_handles_across_worlds(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "readiness": []}) + plan = pr.plan_ports(manifest, instances=2) + credentials = pr.generate_engine_credentials(manifest, token=lambda: "PW") + spawned_argv0s: list[str] = [] + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawned_argv0s.append(argv[0]) + return FakeHandle() + + def fake_sync_run(argv, **kwargs): + return subprocess.CompletedProcess(argv, 0) + + context = pr.SpawnContext( + work_directory=tmp_path, port_plan=plan, credentials=credentials, secret_values={}, + secret_purposes={}, runner=fake_runner, sync_run=fake_sync_run, + ) + world0 = pr.spawn_world(manifest, world_index=0, context=context) + shared = {name: handle for name, handle in world0.handles.items() if plan.is_job_shared(name)} + assert set(shared) == {"postgres"} + spawned_argv0s.clear() + world1 = pr.spawn_world(manifest, world_index=1, context=context, shared_handles=shared) + assert "postgres" not in spawned_argv0s # not respawned for world 1 + assert world1.handles["postgres"] is shared["postgres"] # the very same handle, reused + + +# --- §4.3 healthy() ------------------------------------------------------------------------------ + + +def test_healthy_dispatches_to_the_probe_for_each_declared_readiness_entry() -> None: + manifest = _manifest() # one readiness entry, for `tools` + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + }, + ) + seen: list[tuple[str, int, str | None]] = [] + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen.append((host, port, path)) + return True + + assert pr.probe_runtime_health(manifest, runtime, prober=fake_prober) is True + assert seen == [("localhost", 15001, "/health")] + + +def test_healthy_is_false_when_a_declared_probe_fails() -> None: + manifest = _manifest() + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + }, + ) + assert pr.probe_runtime_health(manifest, runtime, prober=lambda **kwargs: False) is False + + +def test_healthy_is_false_when_a_declared_probes_capability_has_no_endpoint() -> None: + manifest = _manifest() + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, endpoints={}, + ) + assert pr.probe_runtime_health(manifest, runtime, prober=lambda **kwargs: True) is False + + +def test_probe_runtime_health_parses_postgres_credentials_out_of_the_endpoint_address() -> None: + """F9, p5-round1-review: `probe_runtime_health` only ever sees `EnvironmentRuntime`, not the + job's credential map — the rendered + `postgresql://harness:@localhost:/w` address already carries everything a real + probe needs, so it is parsed back out rather than threaded separately.""" + manifest = _manifest( + lambda body: {**body, "readiness": [ + {"capability": "database", "timeout_seconds": 5, "interval_seconds": 0.1}, + ]} + ) + runtime = pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + endpoints={ + "database": pr.RuntimeEndpoint( + capability="database", protocol="postgres", + address="postgresql://harness:s3cr3t@localhost:14000/w0", + configuration_name="DATABASE_URL", + ), + }, + ) + seen: dict[str, Any] = {} + + def fake_prober(*, protocol, host, port, path, user=None, password=None, dbname=None): + seen.update(user=user, password=password, dbname=dbname) + return True + + assert pr.probe_runtime_health(manifest, runtime, prober=fake_prober) is True + assert seen == {"user": "harness", "password": "s3cr3t", "dbname": "w0"} + + +async def _run_healthy(*args: Any, **kwargs: Any) -> bool: + return await pr.healthy(*args, **kwargs) + + +def test_healthy_transitions_follow_section_3_and_never_promote() -> None: + """F6, p5-round1-review — MAJOR (T3). §3's `state` row fixes the legal transitions: + `preparing->ready`, `ready->unhealthy`, and `unhealthy->ready` ONLY via re-provision reconcile + (§4 point 1). `healthy()` is not a reconcile, so it may only ever DEMOTE — this replaces the + old test, which asserted only `preparing->ready` then `ready->unhealthy` in sequence and never + exercised `unhealthy->?` or `stopped->?` at all, so it could not have caught the promotion bug + the old implementation actually had (`RuntimeState.READY if is_healthy else UNHEALTHY`, which + unconditionally promotes ANY prior state to READY on a passing probe).""" + import asyncio + + manifest = _manifest() + endpoints = { + "tools": pr.RuntimeEndpoint( + capability="tools", protocol="http", address="http://localhost:15001", + configuration_name="TOOLS_API_URL", + ), + } + + def make(state: pr.RuntimeState) -> pr.EnvironmentRuntime: + return pr.EnvironmentRuntime( + runtime_id="r1", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=state, endpoints=endpoints, + ) + + preparing = make(pr.RuntimeState.PREPARING) + assert asyncio.run(_run_healthy(manifest, preparing, prober=lambda **kwargs: True)) is True + assert preparing.state is pr.RuntimeState.READY # preparing + healthy -> ready + + still_ready = make(pr.RuntimeState.READY) + assert asyncio.run(_run_healthy(manifest, still_ready, prober=lambda **kwargs: True)) is True + assert still_ready.state is pr.RuntimeState.READY # ready stays ready + + demoted = make(pr.RuntimeState.READY) + assert asyncio.run(_run_healthy(manifest, demoted, prober=lambda **kwargs: False)) is False + assert demoted.state is pr.RuntimeState.UNHEALTHY # ready + unhealthy probe -> demote + + stays_unhealthy = make(pr.RuntimeState.UNHEALTHY) + ok = asyncio.run(_run_healthy(manifest, stays_unhealthy, prober=lambda **kwargs: True)) + assert ok is True # the PROBE passed... + assert stays_unhealthy.state is pr.RuntimeState.UNHEALTHY # ...but state is NOT promoted + + stays_stopped = make(pr.RuntimeState.STOPPED) + asyncio.run(_run_healthy(manifest, stays_stopped, prober=lambda **kwargs: True)) + assert stays_stopped.state is pr.RuntimeState.STOPPED # only provision()/close() clear this + + +# --- default probers: real postgres/http exercise, no fakes -------------------------------------- + + +def test_default_capability_prober_falls_back_to_tcp_when_psycopg_is_absent( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The postgres branch must fall back to a bare TCP probe rather than raising + `ImportError` on hosts without `psycopg`. Absence is simulated by blocking the + module in `sys.modules` — the prober imports at call time, so the import genuinely + fails and the real fallback branch runs regardless of what this venv has installed + (an environment premise check would flip whenever another lane needs `psycopg` + present).""" + monkeypatch.setitem(sys.modules, "psycopg", None) + import socket as socket_module + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + port = server.getsockname()[1] + try: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.POSTGRES, host="localhost", port=port, path=None + ) is True + finally: + server.close() + + +def test_default_capability_prober_tcp_probe_is_false_when_nothing_listens() -> None: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.TCP, host="localhost", port=1, path=None + ) is False + + +def test_probe_postgres_runs_a_real_select_1_when_credentials_are_supplied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """F9, p5-round1-review — MAJOR. Previously connected as `user="postgres", dbname="postgres"` + (neither exists — the catalog's `initdb -U harness` creates only `harness`) and treated almost + any resulting `OperationalError` as "answered, therefore ready." With real credentials, this + must run an actual query — proven with a fake `psycopg` module, since no real server is + available in this lane.""" + executed: list[str] = [] + + class FakeConnection: + def execute(self, query: str) -> None: + executed.append(query) + + def close(self) -> None: + pass + + def fake_connect(**kwargs: Any) -> FakeConnection: + assert kwargs["user"] == "harness" + assert kwargs["dbname"] == "w0" + return FakeConnection() + + fake_module = types.ModuleType("psycopg") + fake_module.connect = fake_connect + fake_module.OperationalError = type("OperationalError", (Exception,), {}) + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + assert pr._probe_postgres( + "localhost", 14000, user="harness", password="pw", dbname="w0" + ) is True + assert executed == ["SELECT 1"] + + +def test_probe_postgres_treats_a_starting_up_server_as_not_ready( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The old substring-match heuristic treated `FATAL: the database system is starting up` + (postgres's OWN response while still in recovery — it binds its listen socket early) as + "answered, therefore ready." A real `SELECT 1` cannot be fooled by it.""" + fake_module = types.ModuleType("psycopg") + operational_error = type("OperationalError", (Exception,), {}) + fake_module.OperationalError = operational_error + + class FakeConnection: + def execute(self, query: str) -> None: + raise operational_error("the database system is starting up") + + def close(self) -> None: + pass + + fake_module.connect = lambda **kwargs: FakeConnection() + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + assert pr._probe_postgres( + "localhost", 14000, user="harness", password="pw", dbname="w0" + ) is False + + +def test_probe_postgres_falls_back_to_tcp_when_credentials_are_not_supplied( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Even when `psycopg` IS available, no generated credentials means no real query is + possible — falls back to the same TCP-only check as the psycopg-absent case, never fabricates + a connection attempt without something to authenticate with.""" + import socket as socket_module + + def fake_connect(**kwargs: Any) -> None: + raise AssertionError("must not attempt to connect without credentials") + + fake_module = types.ModuleType("psycopg") + fake_module.connect = fake_connect + fake_module.OperationalError = Exception + monkeypatch.setitem(sys.modules, "psycopg", fake_module) + + server = socket_module.socket(socket_module.AF_INET, socket_module.SOCK_STREAM) + server.bind(("localhost", 0)) + server.listen(1) + try: + port = server.getsockname()[1] + assert pr._probe_postgres("localhost", port) is True # no user/dbname given + finally: + server.close() + + +def test_probe_http_against_a_real_server_reports_2xx_as_ready() -> None: + """T7, p5-round1-review: `_probe_http` — the default prober for the one capability the shared + manifest actually declares a readiness probe for — had zero direct coverage; every readiness + test in this file injects a fake `prober` instead.""" + import http.server + import threading + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 - stdlib method name + self.send_response(200) + self.end_headers() + self.wfile.write(b"ok") + + def log_message(self, *args: Any) -> None: # silence stderr noise + pass + + server = http.server.HTTPServer(("localhost", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + port = server.server_address[1] + assert pr.default_capability_prober( + protocol=CapabilityProtocol.HTTP, host="localhost", port=port, path="/health" + ) is True + finally: + server.shutdown() + thread.join(timeout=5) + + +def test_probe_http_reports_not_ready_when_nothing_listens() -> None: + assert pr.default_capability_prober( + protocol=CapabilityProtocol.HTTP, host="localhost", port=1, path="/health" + ) is False + + +# ================================================================================================= +# Phase 6: seed application, baseline freeze, world clone, reset, conformance gate, provision/close +# ================================================================================================= +# +# `_manifest()` above already carries one postgres store (`template_database`, job-shared, +# sentinel `SELECT 1` = `"1"`) — reused everywhere below unless a test needs a different strategy, +# in which case a `mutate` callback swaps `seed.stores[0].baseline` (and, where the row content +# under test matters, `migrations`/`seed_files`). `fake_sync_run`/`SqlSpy` below are this section's +# own `ProcessRunner`/`CapabilityProber`-equivalent fakes — no real postgres/redis/rabbitmq/docker +# anywhere, per the module docstring. + + +def _fake_sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + +def _recording_sync_run(calls: list[Any]) -> Callable[..., subprocess.CompletedProcess]: + def run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + return run + + +def _pg_spy_row(value: Any) -> list[tuple[Any, ...]]: + return [(value,)] + + +def _recording_prober(*, fail_times: int = 0) -> Callable[..., bool]: + """t1/N2, p6-review-r2: the promote-path fixtures used to inject `prober=lambda **kwargs: + True` — a fast pass that made `provision()`'s PREPARING->READY promotion and `reset()`'s + post-sentinel probe structurally unable to observe whether either one actually POLLED (N2) or + merely sampled once, the exact class of gap t1 was raised about for `_wait_for_store_ready`. + `fail_times=0` (the default) keeps every EXISTING test's prior instant-pass behavior + unchanged; a test proving the poll passes `fail_times=N` and asserts `len(prober.calls) > 1` + or that state ends up correct only after N+1 calls.""" + calls: list[dict[str, Any]] = [] + remaining = [fail_times] + + def prober(**kwargs: Any) -> bool: + calls.append(kwargs) + if remaining[0] > 0: + remaining[0] -= 1 + return False + return True + + prober.calls = calls # type: ignore[attr-defined] + return prober + + +class SqlSpy: + """A fake `SqlRunner`: records every `(dbname, statement)` call, in order — the "SQL spy" the + task calls for — and simulates just enough real postgres semantics for the baseline-freeze / + world-clone / reset / conformance-gate state machines to run against without a real server: + `CREATE`/`DROP`/`ALTER ... TEMPLATE` database bookkeeping, `to_regclass` existence checks, + `pg_tables`/`COUNT(*)` row counts, and a fixed answer for any `sentinel.query` a test seeds via + `answers`. `canary_leaks` deliberately makes the conformance canary visible across databases, + for the gate's FAIL-path test. + """ + + def __init__( + self, *, answers: dict[str, list[tuple[Any, ...]]] | None = None, canary_leaks: bool = False, + seeded_tables: dict[str, set[str]] | None = None, + ) -> None: + self.calls: list[tuple[str, str]] = [] + self.databases: dict[str, set[str]] = {} + self.row_data: dict[tuple[str, str], int] = {} + self.answers = answers or {} + self.canary_leaks = canary_leaks + # What a freshly `CREATE DATABASE`'d name starts with — simulates "the seed already + # landed" (a real `psql -f ...` would have populated it; the fake `sync_run` this spy + # runs alongside is a structural no-op, per the module docstring). + self.seeded_tables = seeded_tables or {} + + def __call__( + self, *, host: str, port: int, user: str, password: str, dbname: str, statement: str, + read_only: bool = False, + ) -> list[tuple[Any, ...]]: + self.calls.append((dbname, statement)) + body = statement.strip() + if read_only: + # N8, p6-review-r2: simulates postgres's own `SET default_transaction_read_only = on` + # rejecting a non-SELECT statement on a read-only session — the exact class of driver + # error `_call_sql` converts to `store_statement_failed`. Checked PER STATEMENT, not + # just the string's own prefix — psycopg3's simple-query protocol runs every `;`- + # separated statement in one unparameterized `execute()` call, which is exactly how a + # sentinel like `"SELECT 1; DROP TABLE riders"` smuggles a write past a naive + # starts-with-SELECT check while still LOOKING like a read at a glance. + for sub_statement in body.split(";"): + sub = sub_statement.strip() + if sub and not sub.upper().startswith(("SELECT", "WITH")): + raise RuntimeError(f"cannot execute {sub!r} in a read-only transaction") + if body in self.answers: + return self.answers[body] + if body == "SELECT 1": + return [(1,)] # `_manifest()`'s own default sentinel query. + if body.startswith("CREATE DATABASE") and "TEMPLATE" not in body: + name = body.split('"')[1] + self.databases[name] = set(self.seeded_tables.get(name, set())) + return [] + if "TEMPLATE" in body and body.startswith("CREATE DATABASE"): + source, target = body.split('"')[1], body.split('"')[3] + self.databases[source] = set(self.databases.get(target, set())) + return [] + if body.startswith("ALTER DATABASE") and "RENAME TO" in body: + old, new = body.split('"')[1], body.split('"')[3] + self.databases[new] = self.databases.pop(old, set()) + return [] + if body.startswith("DROP DATABASE"): + name = body.split('"')[-2] + self.databases.pop(name, None) + return [] + if body.startswith("CREATE TABLE") and "_alk_conformance" in body: + self.databases.setdefault(dbname, set()).add("_alk_conformance") + if self.canary_leaks: + leaked = "w1" if dbname == "w0" else "w0" + self.databases.setdefault(leaked, set()).add("_alk_conformance") + return [] + if body.startswith("SELECT to_regclass"): + present = "_alk_conformance" in self.databases.get(dbname, set()) + return [(present,)] + if body.startswith("SELECT tablename FROM pg_tables"): + return [(table,) for table in sorted(self.databases.get(dbname, set()))] + if body.startswith("SELECT COUNT(*)"): + table = body.split('"')[1] + return [(self.row_data.get((dbname, table), 0),)] + return [] + + +def _postgres_creds(m: EnvironmentBundleV2) -> dict[str, pr.EngineCredentials]: + return pr.generate_engine_credentials(m, token=lambda: "PW") + + +def _spawn_context( + manifest: EnvironmentBundleV2, *, instances: int = 2, bundle_dir: Path, sql_runner: Any = None, + redis_runner: Any = None, rabbitmq_inspector: Any = None, rabbitmq_declare: Any = None, + rabbitmq_delete: Any = None, rabbitmq_import: Any = None, runner: Any = None, + sync_run: Any = None, prober: Any = None, work_directory: Path, +) -> pr.SpawnContext: + port_plan = pr.plan_ports(manifest, instances=instances) + credentials = _postgres_creds(manifest) + return pr.SpawnContext( + work_directory=work_directory, port_plan=port_plan, credentials=credentials, + secret_values={}, secret_purposes={}, runner=runner or (lambda *a, **k: FakeHandle()), + sync_run=sync_run or _fake_sync_run, sql_runner=sql_runner or SqlSpy(), + redis_runner=redis_runner or (lambda **kwargs: None), + rabbitmq_inspector=rabbitmq_inspector or (lambda **kwargs: 0), + # m8, p6-review-r1: `SpawnContext`'s real defaults for these two now make an actual HTTP + # call — every test lane here must fake them explicitly, same rule as every other engine + # seam in this fixture, so a test that never overrides them never risks a real network + # call regardless of which store the conformance canary ends up preferring. + rabbitmq_declare=rabbitmq_declare or (lambda **kwargs: None), + rabbitmq_delete=rabbitmq_delete or (lambda **kwargs: None), + # N17, p6-review-r2: same rule — seeding now goes over HTTP too. + rabbitmq_import=rabbitmq_import or (lambda **kwargs: None), + bundle_dir=bundle_dir, + # Declared `readiness` probes are P5's own concern (`wait_for_dependency`/`healthy` + # already cover them elsewhere in this file) — a fast-pass fake here so Phase 6's own + # tests, which reuse `_manifest()`'s `tools` readiness entry incidentally, never block on + # a real HTTP probe against a `FakeHandle` that never actually opened a port. Also what + # B4's own `_wait_for_store_ready` polls — fast-pass here means every EXISTING freeze/ + # world-clone test keeps its prior (instant, no-wait) behavior unless it opts into a + # slower fake explicitly (t1's own observability test does exactly that; N2's new + # observability tests do the same one layer up, at the promote sites). + prober=prober or _recording_prober(), + ) + + +# --- §2c seed application ------------------------------------------------------------------------ + + +def test_postgres_seed_argv_shape() -> None: + argv = pr.postgres_seed_argv(port=14000, dbname="alk_baseline_postgres", user="harness", file=Path("db/schema.sql")) + assert argv == [ + "psql", "-h", "localhost", "-p", "14000", "-U", "harness", "-d", "alk_baseline_postgres", + "-v", "ON_ERROR_STOP=1", "-f", "db/schema.sql", + ] + + +def test_redis_seed_argv_has_no_file_flag_stdin_carries_the_commands() -> None: + assert pr.redis_seed_argv(port=15003) == ["redis-cli", "-h", "localhost", "-p", "15003"] + + +def test_default_rabbitmq_definitions_importer_posts_the_file_to_the_definitions_endpoint( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, +) -> None: + """N17, p6-review-r2: `rabbitmqadmin import` is dropped entirely — seeding now POSTs the raw + definitions file straight to the management API's own `/api/definitions` endpoint, the same + HTTP-only seam every other rabbitmq call in this module already uses. No `rabbitmqadmin` + argv/binary anywhere in this path.""" + definitions_file = tmp_path / "mq" / "defs.json" + definitions_file.parent.mkdir(parents=True) + definitions_file.write_bytes(b'{"queues": []}') + creds = pr.EngineCredentials(username="harness", password="pw") + requests: list[Any] = [] + + def fake_urlopen(request: Any, timeout: float | None = None) -> Any: + requests.append(request) + + class _Response: + def __enter__(self) -> "_Response": + return self + + def __exit__(self, *exc: Any) -> None: + return None + + return _Response() + + monkeypatch.setattr(pr.urllib.request, "urlopen", fake_urlopen) + pr.default_rabbitmq_definitions_importer( + host="localhost", port=14002, credentials=creds, file=definitions_file, + ) + assert len(requests) == 1 + assert requests[0].full_url == "http://localhost:24002/api/definitions" + assert requests[0].data == b'{"queues": []}' + assert requests[0].get_header("Authorization") is not None + + +def test_apply_store_seed_runs_migrations_then_seed_files_in_listed_order(tmp_path: Path) -> None: + calls: list[Any] = [] + store = pr.StoreEntry.model_validate({ + "capability": "database", "migrations": ["db/001.sql", "db/002.sql"], + "seed_files": ["db/seed_a.sql", "db/seed_b.sql"], + "baseline": {"strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1"}, + }) + creds = pr.EngineCredentials(username="harness", password="pw") + pr.apply_store_seed( + store, engine=pr.ManagedEngine.POSTGRES, bundle_dir=tmp_path, port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=_recording_sync_run(calls), + ) + files_in_order = [argv[argv.index("-f") + 1] for argv, _ in calls] + assert files_in_order == [ + str(tmp_path / "db/001.sql"), str(tmp_path / "db/002.sql"), + str(tmp_path / "db/seed_a.sql"), str(tmp_path / "db/seed_b.sql"), + ] + + +def test_apply_seed_file_postgres_env_keeps_path_and_adds_pgpassword(tmp_path: Path) -> None: + """A real bug caught by manual exercise before this test existed: `env=` REPLACES a child's + environment, not extends it — passing bare `{"PGPASSWORD": ...}` would drop `PATH` and make + `psql` unfindable outside `subprocess`'s narrow POSIX fallback.""" + calls: list[Any] = [] + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + pr.apply_seed_file( + pr.ManagedEngine.POSTGRES, tmp_path / "db/schema.sql", port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=_recording_sync_run(calls), + ) + _, kwargs = calls[0] + assert kwargs["env"]["PGPASSWORD"] == "s3cr3t" + assert "PATH" in kwargs["env"] + + +def test_apply_seed_file_redis_pipes_file_content_over_stdin(tmp_path: Path) -> None: + seed_file = tmp_path / "cache" / "seed.txt" + seed_file.parent.mkdir(parents=True) + seed_file.write_text("SET greeting hi\n") + calls: list[Any] = [] + pr.apply_seed_file( + pr.ManagedEngine.REDIS, seed_file, port=15003, dbname="", credentials=None, + process_name="cache", sync_run=_recording_sync_run(calls), + ) + argv, kwargs = calls[0] + assert argv == ["redis-cli", "-h", "localhost", "-p", "15003"] + assert kwargs["input"] == "SET greeting hi\n" + + +def test_apply_seed_file_raises_seed_failed_on_nonzero_exit(tmp_path: Path) -> None: + def failing_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + return subprocess.CompletedProcess(argv, 1, stdout="", stderr="syntax error") + + creds = pr.EngineCredentials(username="harness", password="pw") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.apply_seed_file( + pr.ManagedEngine.POSTGRES, tmp_path / "db/broken.sql", port=14000, dbname="x", + credentials=creds, process_name="postgres", sync_run=failing_run, + ) + assert excinfo.value.code == "seed_failed" + assert excinfo.value.stage == "seed" + assert "syntax error" in str(excinfo.value) + + +def test_apply_seed_file_rabbitmq_import_failure_is_seed_failed(tmp_path: Path) -> None: + """rabbitmq's own definitions-import failure is customer-authored seed CONTENT + (§2f `seed_failed`, deterministic, NOT retried), never `store_statement_failed` (the + harness's own statement seam, `infrastructure`, retryable) — the two domains carry opposite + retry semantics, so a code swap here either retries a broken definitions file forever or + gives up on what might have been a transient import blip.""" + creds = pr.EngineCredentials(username="harness", password="pw") + + def failing_import(*, host: str, port: int, credentials: Any, file: Path) -> None: + raise RuntimeError("malformed definitions") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.apply_seed_file( + pr.ManagedEngine.RABBITMQ, tmp_path / "mq" / "defs.json", port=14002, dbname="", + credentials=creds, process_name="mq", sync_run=_fake_sync_run, + rabbitmq_import=failing_import, + ) + assert excinfo.value.code == "seed_failed" + + +# --- §2c sentinel checking ------------------------------------------------------------------------ + + +def test_check_sentinel_postgres_pass_and_fail() -> None: + store = _manifest().seed.stores[0] # {query: "SELECT 1", expected: "1"} + creds = pr.EngineCredentials(username="harness", password="pw") + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=lambda **kwargs: _pg_spy_row(1), + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=lambda **kwargs: _pg_spy_row(0), + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +def test_check_sentinel_passes_read_only_true_to_the_sql_runner() -> None: + """N8, p6-review-r2 (MAJOR): the sentinel is customer-authored content from an untrusted repo, + executed as `harness` — the role `initdb -U harness` makes the postgres SUPERUSER — over a + plain autocommit session. `check_sentinel` must mark this call `read_only=True` so `default_ + sql_runner` puts the session into a read-only transaction before running it.""" + store = _manifest().seed.stores[0] + creds = pr.EngineCredentials(username="harness", password="pw") + seen: dict[str, Any] = {} + + def recording_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + seen.update(kwargs) + return _pg_spy_row(1) + + pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=recording_sql_runner, + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert seen["read_only"] is True + + +def test_a_sentinel_query_attempting_a_write_fails(tmp_path: Path) -> None: + """N8, p6-review-r2 (MAJOR, task-required verification): the door B2 closed for `psql -f` + (dropping privilege for customer-authored SQL) was still open for `sentinel.query` — arbitrary + multi-statement SQL executed as the postgres superuser on a read-write session. `SqlSpy` + (`read_only=True`) rejects a non-`SELECT` statement exactly as a real `SET default_ + transaction_read_only = on` session would; a sentinel that tries to write must surface as a + typed `store_statement_failed`, not silently succeed. + """ + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "sentinel": { + "query": "SELECT 1; DROP TABLE riders", "expected": "1", + }, + }]}} + ) + store = manifest.seed.stores[0] + creds = pr.EngineCredentials(username="harness", password="pw") + sql_spy = SqlSpy() + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.check_sentinel( + store, engine=pr.ManagedEngine.POSTGRES, host="localhost", port=14000, dbname="w0", + credentials=creds, sql_runner=sql_spy, + redis_runner=lambda **kwargs: None, rabbitmq_inspector=lambda **kwargs: 0, + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_call_redis_driver_exception_is_store_statement_failed() -> None: + """`_call_redis` wraps a provisioner-ISSUED command (sentinel/canary probes — + never customer seed content, which goes through `apply_seed_file` instead) — a driver + exception there is the harness's own statement seam, §2f `store_statement_failed` + (`infrastructure`, retryable), never `seed_failed` (`environment`, NOT retried). Swapped, a + transient redis blip during a sentinel check would never retry.""" + def failing_redis_runner(*, host: str, port: int, command: Any) -> Any: + raise RuntimeError("connection reset") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_redis( + failing_redis_runner, stage="conformance", process_name="cache", + host="localhost", port=15003, command=["GET", "x"], + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_call_rabbitmq_driver_exception_is_store_statement_failed() -> None: + """`_call_rabbitmq` wraps a provisioner-ISSUED queue inspection (sentinel/canary probes — + never customer seed content) — a driver exception there is the harness's own statement seam, + §2f `store_statement_failed` (`infrastructure`, retryable), never `seed_failed` + (`environment`, NOT retried). Swapped, a transient rabbitmq blip during a sentinel check would + never retry.""" + def failing_inspector(*, host: str, port: int, credentials: Any, queue: str) -> int: + raise RuntimeError("connection reset") + + creds = pr.EngineCredentials(username="harness", password="pw") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_rabbitmq( + failing_inspector, stage="conformance", process_name="mq", + host="localhost", port=14002, credentials=creds, queue="canary", + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_call_rabbitmq_action_driver_exception_is_store_statement_failed() -> None: + """`_call_rabbitmq_action` wraps the write-side canary declare/publish call — same B5 typing + as `_call_rabbitmq`, against a store that has already passed readiness. A driver exception + there is also §2f `store_statement_failed`, never `seed_failed`.""" + def failing_action(**kwargs: Any) -> None: + raise RuntimeError("connection reset") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._call_rabbitmq_action( + failing_action, stage="conformance", process_name="mq", action="declare", + ) + assert excinfo.value.code == "store_statement_failed" + + +def test_measure_postgres_row_counts_passes_read_only_true(tmp_path: Path) -> None: + """N8, p6-review-r2: the baseline row-count reads (`pg_tables`, `COUNT(*)`) are the other + read call site named in the fix — pinned directly rather than only indirectly through a + freeze-baseline integration test.""" + seen: list[bool] = [] + + def recording_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + seen.append(kwargs["read_only"]) + if kwargs["statement"].startswith("SELECT tablename"): + return [("riders",)] + return [(3,)] + + creds = pr.EngineCredentials(username="harness", password="pw") + counts = pr._measure_postgres_row_counts( + host="localhost", port=14000, credentials=creds, dbname="w0", + sql_runner=recording_sql_runner, process_name="postgres", + ) + assert counts == {"riders": 3} + assert seen == [True, True] # both the table listing AND the COUNT(*) itself. + + +def test_check_sentinel_redis_pass_and_fail() -> None: + store = pr.StoreEntry.model_validate({ + "capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}, + }) + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.REDIS, host="localhost", port=15003, dbname=None, + credentials=None, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: b"hi", + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.REDIS, host="localhost", port=15003, dbname=None, + credentials=None, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +def test_check_sentinel_rabbitmq_pass_and_fail() -> None: + store = pr.StoreEntry.model_validate({ + "capability": "queue", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "c" * 64}, + "sentinel": {"queue": "jobs", "expected_depth": 5}, + }) + creds = pr.EngineCredentials(username="harness", password="pw") + passing = pr.check_sentinel( + store, engine=pr.ManagedEngine.RABBITMQ, host="localhost", port=15003, dbname=None, + credentials=creds, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 5, + ) + assert passing is True + failing = pr.check_sentinel( + store, engine=pr.ManagedEngine.RABBITMQ, host="localhost", port=15003, dbname=None, + credentials=creds, sql_runner=lambda **kwargs: [], redis_runner=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 0, + ) + assert failing is False + + +# --- §5.3 baseline freeze ------------------------------------------------------------------------- + + +def test_freeze_baseline_requires_bundle_dir() -> None: + manifest = _manifest() + ctx = pr.SpawnContext( + work_directory=Path("/x"), port_plan=pr.plan_ports(manifest, instances=1), + credentials={}, secret_values={}, secret_purposes={}, bundle_dir=None, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert excinfo.value.code == "internal_invariant_violated" + + +def test_freeze_baseline_waits_for_readiness_before_the_first_statement(tmp_path: Path) -> None: + """t1, p6-review-r1: B4's fix is structurally UNOBSERVABLE with a prober that always answers + `True` on the first call (`_spawn_context`'s own fast-pass default) — this pins that the wait + is REAL. A prober that fails twice before succeeding must be polled through all of them; the + manifest declares a `database` readiness entry with a tiny interval so the real `time.sleep` + between polls stays in the low milliseconds rather than the 30s/0.25s fallback default. + """ + manifest = _manifest( + lambda body: {**body, "readiness": [ + *body["readiness"], + {"capability": "database", "timeout_seconds": 5, "interval_seconds": 0.01}, + ]} + ) + probe_calls: list[dict[str, Any]] = [] + remaining_failures = [2] + + def flaky_prober(**kwargs: Any) -> bool: + probe_calls.append(kwargs) + if remaining_failures[0] > 0: + remaining_failures[0] -= 1 + return False + return True + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=flaky_prober, work_directory=tmp_path, + ) + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + # 2 failures + the passing call — proves `_wait_for_store_ready` actually looped rather than + # accepting the first `False` as good enough, or skipping the probe entirely. + assert len(probe_calls) >= 3 + assert remaining_failures[0] == 0 + assert all(call["protocol"] is pr.CapabilityProtocol.POSTGRES for call in probe_calls) + + +def test_freeze_baseline_readiness_timeout_is_a_typed_depends_on_timeout(tmp_path: Path) -> None: + """B4, p6-review-r1: a store that never becomes ready must fail typed (`depends_on_timeout`, + §2f — `infrastructure`, retryable), not hang forever or let the next statement run anyway.""" + manifest = _manifest( + lambda body: {**body, "readiness": [ + *body["readiness"], + {"capability": "database", "timeout_seconds": 0.05, "interval_seconds": 0.01}, + ]} + ) + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=lambda **kwargs: False, work_directory=tmp_path, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert excinfo.value.code == "depends_on_timeout" + + +def test_freeze_baseline_template_database_seeds_seals_and_measures_row_counts( + tmp_path: Path, +) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], + "seed_files": ["db/seed.sql"], + }]}} + ) + # `seeded_tables` tells the spy what `CREATE DATABASE "alk_baseline_postgres"` should start + # with — simulating "the seed already landed," since the fake `sync_run` for `psql -f ...` + # below is a structural no-op (same rule as every other engine-binary call in this file). + sql_spy = SqlSpy(seeded_tables={"alk_baseline_postgres": {"riders"}}) + sql_spy.row_data[("alk_baseline_postgres", "riders")] = 3 + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=sql_spy, sync_run=_fake_sync_run, + work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert "postgres" in result.job_shared_handles # stays running — job-shared for the job. + record = result.build_output.stores[0] + assert record.strategy is pr.BaselineStrategy.TEMPLATE_DATABASE + assert record.baseline_reference == "alk_baseline_postgres" + assert record.row_counts == {"riders": 3} + + statements = [statement for _, statement in sql_spy.calls] + assert any(s.startswith("CREATE DATABASE") and "TEMPLATE" not in s for s in statements) + # migrations/seed_files apply via `sync_run`, not `sql_runner` — nothing to find in `statements` + # for them; asserting the CREATE precedes the ALTER is what is left to check here. + create_index = next(i for i, s in enumerate(statements) if s.startswith("CREATE DATABASE")) + alter_index = next(i for i, s in enumerate(statements) if "IS_TEMPLATE" in s) + assert create_index < alter_index + assert statements[alter_index] == ( + 'ALTER DATABASE "alk_baseline_postgres" WITH IS_TEMPLATE true ALLOW_CONNECTIONS false' + ) + + +def test_freeze_baseline_seed_commands_run_under_the_stores_declared_user(tmp_path: Path) -> None: + """t4 / B2, p6-review-r1: migration/seed files are customer-authored content applied through + `psql -f`, which honors backslash meta-commands (`\\!`, `\\copy ... program`) — must run under + the store's declared `svc-data` identity, never the provisioner's own uid, the same privilege + drop every other untrusted-content execution path in this module already gets + (`build_commands`, the managed-engine daemon itself).""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], "seed_files": [], + }]}} + ) + (tmp_path / "db").mkdir() + (tmp_path / "db" / "schema.sql").write_text("-- schema\n") + seed_calls: list[Any] = [] + + def recording_sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=recording_sync_run, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx = dc_replace( + ctx, user_resolver=_fake_user_resolver({"svc-data": (4444, 5555)}), + chown=lambda path, uid, gid: None, # a fake uid needs no real os.chown to prove the point. + ) + + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + psql_calls = [(argv, kwargs) for argv, kwargs in seed_calls if argv and argv[0] == "psql"] + assert psql_calls, "expected a psql -f seed invocation" + _, kwargs = psql_calls[0] + assert kwargs.get("user") == 4444 + assert kwargs.get("group") == 5555 + + +def test_freeze_baseline_datadir_copy_terminates_the_bootstrap_and_snapshots_the_datadir( + tmp_path: Path, +) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + handles: list[FakeHandle] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + handle = FakeHandle() + handles.append(handle) + return handle + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert result.job_shared_handles == {} # datadir_copy is never job-shared. + assert len(handles) == 1 + assert handles[0].terminated is True # the bootstrap instance is stopped after the copy. + record = result.build_output.stores[0] + assert record.strategy is pr.BaselineStrategy.DATADIR_COPY + assert Path(record.baseline_reference) == tmp_path / "managed" / "postgres.baseline" + + +def test_freeze_baseline_datadir_copy_redis_issues_save_before_terminating( + tmp_path: Path, +) -> None: + """Q1, p6-review-r3 (MAJOR): `redis_daemon_argv` disables save points (`--save ""`) — a bare + SIGTERM shutdown persists nothing, so `_freeze_one_store`'s `DATADIR_COPY` branch must issue + an explicit synchronous `SAVE` immediately before terminating, or the copied data dir every + world clones/resets from is an empty baseline.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + events: list[str] = [] + + class RecordingHandle(FakeHandle): + def terminate(self) -> None: + events.append("TERMINATE") + super().terminate() + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + return RecordingHandle() + + def redis_runner(*, host: str, port: int, command: Any) -> Any: + events.append(command[0]) + return "hi" if command[0] == "GET" else None + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, redis_runner=redis_runner, + work_directory=tmp_path, + ) + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + # only "cache" (redis, datadir_copy) is ever terminated at freeze time — postgres stays + # running (template_database, job-shared), so this is unambiguously the redis command order. + assert events[-2:] == ["SAVE", "TERMINATE"] + + +def test_freeze_baseline_empty_strategy_is_a_no_op_capture(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + spawned_names: list[str] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + return FakeHandle() + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + spawned_names.append(argv[0]) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, runner=runner, sync_run=sync_run, work_directory=tmp_path, + ) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + cache_record = next(r for r in result.build_output.stores if r.process_name == "cache") + assert cache_record.strategy is pr.BaselineStrategy.EMPTY + assert cache_record.baseline_reference == "" + assert cache_record.row_counts == {} + assert "redis-cli" not in spawned_names # nothing seeded at freeze time for `empty`. + assert "cache" not in result.job_shared_handles + + +def test_write_build_output_shape(tmp_path: Path) -> None: + build_output = pr.BuildOutput( + bundle_digest="sha256:" + "0" * 64, + stores=[pr.StoreBaselineRecord( + capability="database", process_name="postgres", engine=pr.ManagedEngine.POSTGRES, + strategy=pr.BaselineStrategy.TEMPLATE_DATABASE, inputs_digest="sha256:" + "a" * 64, + baseline_reference="alk_baseline_postgres", row_counts={"riders": 3}, + )], + conformance=True, conformance_reason=None, + ) + target = pr.write_build_output(tmp_path, build_output) + assert target == tmp_path / "artifacts" / "build.json" + payload = json.loads(target.read_text()) + assert payload["bundle_digest"] == build_output.bundle_digest + assert payload["conformance"] is True + assert payload["stores"] == [{ + "capability": "database", "process_name": "postgres", "engine": "postgres", + "strategy": "template_database", "inputs_digest": "sha256:" + "a" * 64, + "baseline_reference": "alk_baseline_postgres", "row_counts": {"riders": 3}, + }] + + +# --- §4.2 world clone + reset ---------------------------------------------------------------------- + + +def _frozen_template_database( + manifest: EnvironmentBundleV2, tmp_path: Path, +) -> tuple[pr.FreezeResult, pr.SpawnContext, SqlSpy]: + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + sql_spy.calls.clear() + return result, ctx, sql_spy + + +def test_world_clone_template_database_issues_terminate_drop_create_template( + tmp_path: Path, +) -> None: + manifest = _manifest() + freeze_result, ctx, sql_spy = _frozen_template_database(manifest, tmp_path) + + result = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert set(result.handles) == {"postgres", "tools-api", "agent"} + assert result.handles["postgres"] is freeze_result.job_shared_handles["postgres"] # reused. + + statements = [statement for _, statement in sql_spy.calls] + assert statements == [ + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity " + "WHERE datname = 'w0' AND pid <> pg_backend_pid()", + 'DROP DATABASE IF EXISTS "w0"', + 'CREATE DATABASE "w0" TEMPLATE "alk_baseline_postgres"', + ] + + +def test_world_clone_datadir_copy_copies_the_baseline_then_renames_to_wn(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + baseline_dir = Path(freeze_result.build_output.stores[0].baseline_reference) + assert baseline_dir.is_dir() + sql_spy.calls.clear() + + copy_calls: list[tuple[Path, Path]] = [] + real_copy = pr._copytree_preserving_symlinks + + def recording_copy(src: Path, dst: Path) -> None: + copy_calls.append((src, dst)) + real_copy(src, dst) + + ctx2 = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx2 = dc_replace(ctx2, copy=recording_copy) + + result = pr._clone_or_reset_world( + manifest, 0, context=ctx2, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert copy_calls == [(baseline_dir, tmp_path / "worlds" / "w0" / "postgres")] + statements = [statement for _, statement in sql_spy.calls] + assert statements == ['ALTER DATABASE "alk_baseline_postgres" RENAME TO "w0"'] + assert "postgres" in result.handles + + +def test_world_clone_empty_strategy_reseeds_on_every_clone(tmp_path: Path) -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + seed_calls: list[Any] = [] + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append(argv) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=sync_run, work_directory=tmp_path, + ) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + assert not any("redis-cli" in argv for argv in seed_calls) # nothing at freeze time. + + pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert any("redis-cli" in argv for argv in seed_calls) # (re)established at clone time. + + +def test_world_clone_empty_strategy_reseed_runs_under_the_stores_declared_user( + tmp_path: Path, +) -> None: + """t4 / B2, p6-review-r1: the `empty`-strategy reseed path (`_seal_world_store`'s fallback + branch) is a SEPARATE call site from `freeze_baseline`'s own seed application — both must + drop privilege, not just the one exercised by the freeze-time test above.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + seed_calls: list[Any] = [] + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + seed_calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sync_run=sync_run, work_directory=tmp_path, + ) + from dataclasses import replace as dc_replace + ctx = dc_replace( + ctx, user_resolver=_fake_user_resolver({"svc-data": (6666, 7777)}), + chown=lambda path, uid, gid: None, # a fake uid needs no real os.chown to prove the point. + ) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + redis_calls = [(argv, kwargs) for argv, kwargs in seed_calls if "redis-cli" in argv] + assert redis_calls + _, kwargs = redis_calls[-1] + assert kwargs.get("user") == 6666 + assert kwargs.get("group") == 7777 + + +def test_world_clone_empty_strategy_requires_bundle_dir_to_reseed(tmp_path: Path) -> None: + """Mirrors `freeze_baseline`'s own `bundle_dir` guard: an empty-strategy store re-seeded on + every clone must fail typed if the context somehow carries no bundle directory, never silently + resolve `seed_files` against the wrong (cwd) directory.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + (tmp_path / "cache").mkdir(parents=True, exist_ok=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting hi\n") + ctx = _spawn_context(manifest, bundle_dir=tmp_path, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + from dataclasses import replace as dc_replace + ctx_no_bundle_dir = dc_replace(ctx, bundle_dir=None) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._clone_or_reset_world( + manifest, 0, context=ctx_no_bundle_dir, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + assert excinfo.value.code == "internal_invariant_violated" + + +def test_reset_world_terminates_only_per_world_handles(tmp_path: Path) -> None: + manifest = _manifest() + freeze_result, ctx, sql_spy = _frozen_template_database(manifest, tmp_path) + world0 = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + old_tools_api = world0.handles["tools-api"] + old_agent = world0.handles["agent"] + shared_postgres = world0.handles["postgres"] + + handles, healthy = pr.reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles=world0.handles, + ) + assert old_tools_api.handle.terminated is True + assert old_agent.handle.terminated is True + assert shared_postgres.handle.terminated is False # job-shared — stays up across a reset. + assert healthy is True # SqlSpy answers `SELECT 1` -> 1 by default (no `answers` override). + + +def test_reset_world_sentinel_failure_reports_unhealthy_without_raising(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy() # passes at freeze time (default "SELECT 1" -> 1) — m3's own freeze-time + # sentinel check must NOT be what fails this test; only `reset_world`'s is under test here. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world0 = pr._clone_or_reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + # Corrupted AFTER the (passing) freeze-time check, simulating something breaking the world's + # own state before its reset — never matches the sentinel's "1" from here on. + sql_spy.answers["SELECT 1"] = [(0,)] + handles, healthy = pr.reset_world( + manifest, 0, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles=world0.handles, + ) + assert healthy is False # §4.2: a sentinel failure is reported, never raised. + + +# --- §4 conformance gate ------------------------------------------------------------------------- + + +def test_first_canary_store_prefers_postgres_over_redis_and_rabbitmq() -> None: + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + {"capability": "cache", "migrations": [], "seed_files": [], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "k", "expected": "v"}}, + body["seed"]["stores"][0], + ]}} + ) + store = pr._first_canary_store(manifest) + assert store is not None + assert store.capability == "database" # postgres, even though redis is listed first. + + +def test_first_canary_store_is_none_with_no_seed_block() -> None: + manifest = _manifest(lambda body: {**body, "runtime": {**body["runtime"], "kind": "external"}, + "processes": [], "seed": None}) + assert pr._first_canary_store(manifest) is None + + +def _manifest_with_rabbitmq_store() -> EnvironmentBundleV2: + return _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "queue", "kind": "managed", "engine": "rabbitmq", "version": "3.13", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "queue": {"protocol": "amqp", "service": "queue", "configuration_name": "QUEUE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "queue", "migrations": [], "seed_files": [], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "d" * 64}, + "sentinel": {"queue": "jobs", "expected_depth": 0}}, + ]}} + ) + + +def test_run_canary_probe_rabbitmq_declares_publishes_and_inspects_without_deleting( + tmp_path: Path, +) -> None: + """m8, p6-review-r1: previously only ever INSPECTED a queue nobody had declared — a vacuous + pass regardless of real isolation. Pins the real sequence: declare+publish in world 0, inspect + world 1 — real HTTP calls faked, never skipped. + + N15, p6-review-r2 (MINOR): no delete anymore. The delete used to run HERE, before `run_ + conformance_gate`'s own `reset_world` calls for both worlds — by the time `_verify_canary_ + absent` later re-checked world 0, the queue was already gone from THIS unrelated step, + proving nothing about whether the reset itself actually worked. Cleanup is now the reset's + job (world 0's rabbitmq is always `datadir_copy`, wiped and restarted from the pristine + baseline snapshot on every reset), the same mechanism postgres/redis's own canary already + relied on.""" + manifest = _manifest_with_rabbitmq_store() + store = manifest.seed.stores[1] # the rabbitmq store. + declare_calls: list[dict[str, Any]] = [] + delete_calls: list[dict[str, Any]] = [] + inspected_ports: list[int] = [] + + def rabbitmq_declare(**kwargs: Any) -> None: + declare_calls.append(kwargs) + + def rabbitmq_delete(**kwargs: Any) -> None: + delete_calls.append(kwargs) + + def rabbitmq_inspector(**kwargs: Any) -> int: + inspected_ports.append(kwargs["port"]) + return 0 # world 1 never saw the canary — real isolation. + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, work_directory=tmp_path, + rabbitmq_declare=rabbitmq_declare, rabbitmq_delete=rabbitmq_delete, + rabbitmq_inspector=rabbitmq_inspector, + ) + result = pr._run_canary_probe(manifest, store, pr.ManagedEngine.RABBITMQ, context=ctx) + assert result is True + assert len(declare_calls) == 1 + assert declare_calls[0]["queue"] == "_alk_conformance" + assert delete_calls == [] # N15: cleanup deferred to the subsequent reset, not done here. + port0 = ctx.port_plan.port_for("queue", 0) + port1 = ctx.port_plan.port_for("queue", 1) + assert declare_calls[0]["port"] == port0 # declared in world 0. + assert inspected_ports == [port1] # inspected in world 1. + + +def test_run_canary_probe_rabbitmq_fails_when_the_queue_leaks_to_world_1(tmp_path: Path) -> None: + """m8: a leaked canary (world 1's inspector reports a message that should not be there) must + fail the probe, proving the check is not vacuous in the other direction either.""" + manifest = _manifest_with_rabbitmq_store() + store = manifest.seed.stores[1] + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, work_directory=tmp_path, + rabbitmq_declare=lambda **kwargs: None, rabbitmq_delete=lambda **kwargs: None, + rabbitmq_inspector=lambda **kwargs: 1, # world 1 sees a message: a real leak. + ) + result = pr._run_canary_probe(manifest, store, pr.ManagedEngine.RABBITMQ, context=ctx) + assert result is False + + +def test_default_rabbitmq_queue_inspector_treats_404_as_zero_depth( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """m8, p6-review-r1: a 404 for a nonexistent queue IS "empty" for the canary/sentinel's own + purposes — the management API's way of saying so, not an error the gate's 'never raises' + promise should have had to survive only by luck (unreachable via postgres-first preference).""" + import urllib.error + + def fake_urlopen(request: Any, timeout: float | None = None) -> Any: + raise urllib.error.HTTPError(request.full_url, 404, "Not Found", None, None) + + monkeypatch.setattr(pr.urllib.request, "urlopen", fake_urlopen) + creds = pr.EngineCredentials(username="harness", password="pw") + depth = pr.default_rabbitmq_queue_inspector( + host="localhost", port=15003, credentials=creds, queue="_alk_conformance", + ) + assert depth == 0 + + +def test_conformance_gate_passes_when_worlds_are_really_isolated(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy() # `canary_leaks=False` — the default; a real isolation bug would leak. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (True, None) + + +def test_conformance_gate_fails_and_never_raises_when_isolation_is_broken(tmp_path: Path) -> None: + manifest = _manifest() + sql_spy = SqlSpy(canary_leaks=True) # simulates a real cross-world isolation bug. + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +def test_conformance_gate_is_vacuously_true_with_no_canary_store(tmp_path: Path) -> None: + manifest = _manifest(lambda body: {**body, "runtime": {**body["runtime"], "kind": "external"}, + "processes": [], "seed": None}) + ctx = _spawn_context(manifest, bundle_dir=tmp_path, work_directory=tmp_path) + build_output = pr.BuildOutput(bundle_digest=manifest.digest, stores=[]) + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=build_output, job_shared_handles={}, world_handles={}, + ) + assert (passed, reason) == (True, None) + + +class _StickyCanarySqlSpy(SqlSpy): + """Simulates a RESET that runs (DROP/CREATE TEMPLATE are still recorded + normally, so the isolation probe's own w0-vs-w1 read is unaffected) but fails to actually + clear the conformance canary from the world it was planted in — `_alk_conformance` stays + `to_regclass`-visible in that one database no matter how many times it gets dropped and + recreated. Proves `_verify_canary_absent` is load-bearing, not redundant with the per-world + declared-sentinel check (`SELECT 1`), which never queries the canary table at all.""" + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._sticky_dbname: str | None = None + + def __call__(self, **kwargs: Any) -> list[tuple[Any, ...]]: + statement = kwargs["statement"].strip() + if statement.startswith("CREATE TABLE") and "_alk_conformance" in statement: + self._sticky_dbname = kwargs["dbname"] + rows = super().__call__(**kwargs) + if statement.startswith("SELECT to_regclass") and kwargs["dbname"] == self._sticky_dbname: + return [(True,)] + return rows + + +def test_conformance_gate_fails_when_reset_leaves_the_canary_behind(tmp_path: Path) -> None: + """A reset that (for whatever reason) fails to actually clear the reserved + `_alk_conformance` object must fail the gate via `_verify_canary_absent` — this is the + project's own named CRITICAL calibration example, "vacuous canary pass" (severity-grading.md). + Isolation and both worlds' own declared sentinels pass; only the post-reset canary-absence + check catches it.""" + manifest = _manifest() + sql_spy = _StickyCanarySqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +class _BadResetSentinelSqlSpy(SqlSpy): + """The store's OWN declared sentinel (`SELECT 1`) answers correctly at freeze + time (queried against the baseline database) but wrong against either world database + (`w0`/`w1`) — simulating a reset whose reseal left the declared state broken, which + `reset_world`'s `_check_all_sentinels` is supposed to catch and the gate is supposed to + escalate via `sentinel_ok`, distinct from (and reached before) the canary-absence check.""" + + def __call__(self, **kwargs: Any) -> list[tuple[Any, ...]]: + if kwargs["statement"].strip() == "SELECT 1" and kwargs["dbname"] in ("w0", "w1"): + return [(0,)] + return super().__call__(**kwargs) + + +def test_conformance_gate_fails_when_a_worlds_own_sentinel_fails_after_reset( + tmp_path: Path, +) -> None: + """`sentinel_ok` from the gate's own per-world `reset_world` calls must actually + gate the result — a broken reset that fails the store's declared sentinel must degrade + parallelism, not be silently overridden by an otherwise-clean canary-absence check.""" + manifest = _manifest() + sql_spy = _BadResetSentinelSqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + passed, reason = pr.run_conformance_gate( + manifest, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +# --- §4 provision / reset / close: ProcessRuntimeProvider ------------------------------------------- + + +def _sql_spy_provider(**overrides: Any) -> pr.ProcessRuntimeProvider: + kwargs: dict[str, Any] = dict( + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, sql_runner=SqlSpy(), + prober=_recording_prober(), # N2, p6-review-r2: see `_spawn_context`'s own comment. + rabbitmq_declare=lambda **kwargs: None, rabbitmq_delete=lambda **kwargs: None, + rabbitmq_import=lambda **kwargs: None, + ) + kwargs.update(overrides) + return pr.ProcessRuntimeProvider(**kwargs) + + +def _provision_dirs(tmp_path: Path) -> tuple[Path, Path]: + """B1, p6-review-r1: `source` (the untrusted checkout) and `bundle_dir` (the verified bundle + root) are now GENUINELY DISTINCT directories, never the same path passed twice — a regression + back to conflating them would be invisible if every test fixture kept them identical (t2).""" + source = tmp_path / "source" + (source / "services" / "tools-api").mkdir(parents=True) + bundle_dir = tmp_path / "bundle" + bundle_dir.mkdir(parents=True) + return source, bundle_dir + + +def test_provision_reconciles_to_exactly_w_ready_worlds(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0, 1, 2] + assert len({runtime.runtime_id for runtime in runtimes}) == 3 # never duplicated. + # t3, p6-review-r1: §4.1 says `provision` "reconciles to exactly `instances` READY worlds" — + # previously true only of the COUNT, never checked that a returned world is actually `ready` + # (it was `preparing`, forever, until some OTHER caller happened to call `healthy()`). + assert all(runtime.state is pr.RuntimeState.READY for runtime in runtimes) + ports = {runtime.endpoints["tools"].address for runtime in runtimes} + assert len(ports) == 3 # each world's own tools-api port, all distinct. + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["conformance"] is True + # m1, p6-review-r1: both degrade causes now land on build.json, not just the gate's own. + assert build_output["requested_parallelism"] == 3 + assert build_output["effective_parallelism"] == 3 + assert build_output["degrade_reason"] is None + + +def test_provision_fixed_port_at_w1_records_no_degrade(tmp_path: Path) -> None: + """`fixed_port` forces `effective_instances=1` regardless of `instances` — + at `instances=1` that's not a degrade, since requested==effective already. A prior bug copied + `PortPlan.degraded_reason` verbatim onto `build.json` here, so a job that asked for W=1 and + got W=1 reported a parallelism degrade that never happened (crashes `hosted_entrypoint`'s + `parallelism_degraded` emission downstream, whose payload requires `effective < requested`).""" + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 1 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] is None + + +def test_provision_fixed_port_above_w1_records_degrade(tmp_path: Path) -> None: + """Companion to the test above: at `instances>1` the same `fixed_port` constraint IS a real + degrade (`effective=1 < requested=3`), so `build.json` must still report it.""" + manifest = _manifest( + lambda body: { + **body, + "processes": [ + body["processes"][0], {**body["processes"][1], "fixed_port": 8081}, + body["processes"][2], + ], + } + ) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [runtime.world_index for runtime in runtimes] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 3 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] == "fixed_port" + + +def test_provision_reads_seed_files_from_bundle_dir_not_source(tmp_path: Path) -> None: + """B1, p6-review-r1: §2c seed/migration paths are bundle-relative and must resolve against + the VERIFIED bundle directory, never the untrusted checkout — a bundle declares `migrations: + ["db/schema.sql"]`; preflight hashes/scans `/db/schema.sql`, so reading the same + relative path from `source` instead would execute bytes nothing ever verified. `source` and + `bundle_dir` are genuinely different directories and the seed file exists ONLY under + `bundle_dir` (t2) — a regression back to `bundle_dir=source` is caught by asserting the + RECORDED `-f` argument, not merely that the fake `sync_run` returned success (which it always + does regardless of the path).""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], "migrations": ["db/schema.sql"], "seed_files": [], + }]}} + ) + source, bundle_dir = _provision_dirs(tmp_path) + (bundle_dir / "db").mkdir(parents=True) + (bundle_dir / "db" / "schema.sql").write_text("-- schema\n") + + calls: list[Any] = [] + provider = _sql_spy_provider( + sync_run=_recording_sync_run(calls), secrets_path=tmp_path / "secrets.json", + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + seed_calls = [argv for argv, _ in calls if argv and argv[0] == "psql"] + assert seed_calls, "expected a psql -f seed invocation" + applied_file = seed_calls[0][seed_calls[0].index("-f") + 1] + assert applied_file == str(bundle_dir / "db" / "schema.sql") + + +def test_provision_empty_strategy_first_clone_reads_seed_files_from_bundle_dir_not_work_directory( + tmp_path: Path, +) -> None: + """Mirrors the test above for the OTHER `apply_store_seed` call site — `_seal_world_store`'s + `empty`-strategy branch, reached on every world clone/reset, not just `freeze_baseline`'s + once-per-job seed. `strategy: empty` never seeds at freeze (§5.3's own no-op capture; postgres + itself does not support it, so a redis `cache` store — the same shape the pre-existing + empty-strategy fixtures use — isolates this second call site cleanly). Every existing + `empty`-strategy fixture used `bundle_dir == work_directory`, so a regression swapping the two + there was invisible — genuinely distinct directories here, same proof technique as the + freeze-path test above. A decoy file of the SAME NAME is planted under `work_directory` too, + with different content — asserting content, not just presence, is what actually proves the + code read from `bundle_dir` rather than merely finding a same-named file wherever it looked.""" + manifest = _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "cache", "kind": "managed", "engine": "redis", "version": "7", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "cache": {"protocol": "redis", "service": "cache", "configuration_name": "CACHE_URL"}, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "cache", "migrations": [], "seed_files": ["cache/seed.txt"], + "baseline": {"strategy": "empty", "inputs_digest": "sha256:" + "b" * 64}, + "sentinel": {"key": "greeting", "expected": "hi"}}, + ]}} + ) + source, bundle_dir = _provision_dirs(tmp_path) + (bundle_dir / "cache").mkdir(parents=True) + (bundle_dir / "cache" / "seed.txt").write_text("SET greeting hi\n") + (tmp_path / "cache").mkdir(parents=True) + (tmp_path / "cache" / "seed.txt").write_text("SET greeting DECOY\n") + + calls: list[Any] = [] + provider = _sql_spy_provider( + sync_run=_recording_sync_run(calls), secrets_path=tmp_path / "secrets.json", + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + redis_calls = [(argv, kwargs) for argv, kwargs in calls if argv and "redis-cli" in argv] + assert redis_calls, "expected a redis-cli seed invocation from the empty-strategy RESET path" + assert redis_calls[0][1]["input"] == "SET greeting hi\n" + + +def test_provision_is_idempotent_for_the_same_job_identity(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + spawn_count: list[int] = [0] + + def counting_runner(argv, *, cwd, env, log_path, user=None, group=None): + spawn_count[0] += 1 + return FakeHandle() + + provider = _sql_spy_provider(runner=counting_runner, secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + count_after_first = spawn_count[0] + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.runtime_id for r in first] == [r.runtime_id for r in second] + assert spawn_count[0] == count_after_first # nothing re-spawned; every world already `ready`. + + +def test_provision_hosted_mode_require_declared_user_true_fails_typed_when_unresolvable( + tmp_path: Path, +) -> None: + """P5 ledger carry-forward, exercised end to end through `provision()` rather than only at + `build_process_tree`/`spawn_source_process` directly: the hosted path passes + `require_declared_user=True`, and a snapshot that somehow lacks a declared `svc-*` user must + fail typed, never silently fall back to running unprivileged. M2, p6-review-r1: `True` is now + `provision()`'s own DEFAULT (fail-closed) — passed explicitly here anyway so the test still + reads as pinning the behavior on its own, not merely relying on the default not having moved.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + user_resolver=lambda name: None, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=True, + )) + assert excinfo.value.code == "spawn_failed" + + +def test_provision_require_declared_user_defaults_to_true(tmp_path: Path) -> None: + """M2, p6-review-r1: `ProcessRuntimeProvider` is hosted-only — a caller that omits the keyword + entirely must still fail closed, not silently fall back to running everything unprivileged as + the harness's own `svc-control`. Mirrors the test above but never passes the argument.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + user_resolver=lambda name: None, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + )) + assert excinfo.value.code == "spawn_failed" + + +def test_provision_conformance_degrade_persists_across_a_later_reconcile_call( + tmp_path: Path, +) -> None: + """A real bug caught by manual exercise before this test existed: `effective` is recomputed + fresh from `port_plan` on every call, so a gate failure decided on call 1 was silently + forgotten on call 2 — worlds beyond index 0 came back even though the gate never re-ran.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + sql_runner=SqlSpy(canary_leaks=True), secrets_path=tmp_path / "secrets.json", + ) + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in first] == [0] + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in second] == [0] + assert first[0].runtime_id == second[0].runtime_id + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["degrade_reason"] == "conformance_gate_failed" # m1 + + +def test_provision_reconcile_at_w1_after_gate_failure_records_no_degrade(tmp_path: Path) -> None: + """A sick-world recovery re-call can legitimately pass a smaller `instances` than the job's + original request (the module's own comment above the reconcile branch). The sticky + conformance-degrade branch does not know the current call's `requested` — at `instances=1`, + `effective == requested == 1` already, so re-surfacing the earlier gate failure there would + write `build.json` a `parallelism_degraded`-shaped record with no valid payload + (`effective < requested` is required, and 1 < 1 is false).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider( + sql_runner=SqlSpy(canary_leaks=True), secrets_path=tmp_path / "secrets.json", + ) + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=3, + require_declared_user=False, + )) + assert [r.world_index for r in first] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["degrade_reason"] == "conformance_gate_failed" + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert [r.world_index for r in second] == [0] + build_output = json.loads((tmp_path / "artifacts" / "build.json").read_text()) + assert build_output["requested_parallelism"] == 1 + assert build_output["effective_parallelism"] == 1 + assert build_output["degrade_reason"] is None + + +def test_provision_tears_down_before_rebuilding_on_a_bundle_digest_change(tmp_path: Path) -> None: + """M6, p6-review-r1: a bundle-digest change (a re-sealed bundle mid-attempt) used to reassign + this instance's own identity straight over the PREVIOUS job's still-running processes and + still-allocated ports — §4.1's "never duplicates" broken in the one case this branch exists + for. Every previously-live handle (job-shared and per-world) must be terminated BEFORE the new + manifest's own build/freeze ever runs.""" + manifest_a = _manifest() + manifest_b = _manifest(lambda body: {**body, "digest": "sha256:" + "9" * 64}) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + + first = asyncio.run(provider.provision( + manifest_a, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + old_world_handles = [h for world in provider._world_handles.values() for h in world.values()] + old_shared_handles = list(provider._job_shared_handles.values()) + assert old_world_handles or old_shared_handles # something is actually running to tear down. + + second = asyncio.run(provider.provision( + manifest_b, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert all(h.handle.terminated for h in old_world_handles) + assert all(h.handle.terminated for h in old_shared_handles) + assert second[0].bundle_digest == manifest_b.digest + assert second[0].bundle_digest != first[0].bundle_digest + + +def test_provision_recovers_a_sick_world_via_re_call(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = first[0] + old_agent_handle = provider._world_handles[0]["agent"] + old_runtime_id = runtime.runtime_id # N1, p6-review-r2: snapshotted BEFORE mutation — after + # N1, `second[0]` and `runtime` are the SAME object, so comparing `second[0].runtime_id != + # runtime.runtime_id` post-rebuild would compare an attribute against itself. + runtime.state = pr.RuntimeState.UNHEALTHY + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert second[0] is runtime # N1: mutated in place, never replaced. + assert second[0].runtime_id != old_runtime_id # rebuilt, not left unhealthy. + # t3, p6-review-r1: `provision()` now promotes a freshly-(re)built world out of `PREPARING` + # before returning (the same declared-readiness probe `healthy()` uses) — was `PREPARING` + # before this fix pass, unconditionally. + assert second[0].state is pr.RuntimeState.READY + assert old_agent_handle.handle.terminated is True + + +def test_reset_transitions_ready_or_unhealthy_per_sentinel(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + asyncio.run(provider.reset(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.READY + + +def test_ensure_world_mutates_the_same_environment_runtime_object_across_rebuilds( + tmp_path: Path, +) -> None: + """N1, p6-review-r2 (BLOCKER), superseding m5's own test: v1.12 §4.5b's live-object model + ("providers hand out live `EnvironmentRuntime` objects") reads as ONE object per world for + the provider's whole life. `_ensure_world` used to mint a brand-new `EnvironmentRuntime` on + every rebuild (m5's own fixture exercised exactly that, holding a deliberately STALE, + replaced object) — under that shape, a caller holding an EARLIER reference (e.g. `hosted_ + scheduler.py`'s own pool entry, captured right after the first `provision()` call) could never + see a later rebuild's state land anywhere it could observe. Pins that a sick-world recovery + rebuild MUTATES the object a caller is already holding, in place — fresh `runtime_id`/ + `endpoints`/`state`, same Python object, visible through the ORIGINAL reference with no + re-fetch required. `reset()`'s own state write (m5) is a trivial consequence once there is + only one object to write to. + """ + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + first = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + held = first[0] + old_runtime_id = held.runtime_id + held.state = pr.RuntimeState.UNHEALTHY # simulate the scheduler demoting it (§4.5b). + + second = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert second[0] is held # the SAME object — never replaced. + assert held.runtime_id != old_runtime_id # rebuilt: a fresh identity... + assert held.state is pr.RuntimeState.READY # ...and promoted — both visible through `held`. + + asyncio.run(provider.reset(held, work_directory=tmp_path)) + assert provider._runtimes[0] is held # reset() writes through the SAME object too. + assert held.state is pr.RuntimeState.READY + + +def test_close_is_idempotent_and_removes_secrets_and_data_directories(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "run-secrets.json" + secrets_path.write_text("{}") + provider = _sql_spy_provider(secrets_path=secrets_path) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + handles = [handle for world in provider._world_handles.values() for handle in world.values()] + assert handles # something is actually running before close(). + # B3, p6-review-r1: the secrets file is gone the moment `provision()` loaded it — long before + # `close()` ever runs (t5). `close()`'s own unlink is the "if still present" backstop only. + assert not secrets_path.exists() + + asyncio.run(provider.close(work_directory=tmp_path)) + assert all(handle.handle.terminated for handle in handles) + assert not secrets_path.exists() + assert not (tmp_path / "build").exists() + assert not (tmp_path / "worlds").exists() + assert not (tmp_path / "managed").exists() + for runtime in runtimes: + assert runtime.state is pr.RuntimeState.STOPPED + + asyncio.run(provider.close(work_directory=tmp_path)) # must not raise the second time. + + +def test_close_removes_a_secrets_file_left_behind_by_a_failed_load(tmp_path: Path) -> None: + """`_load_and_delete_secrets` only unlinks `secrets.json` AFTER `json.loads` succeeds — a + malformed file raises `secrets/spawn_failed` with the file still on disk. Every OTHER test + that reaches `close()` does so via a successful load, where the file is already gone by the + time `close()` runs, so its own unlink line is otherwise a no-op across the whole suite; + plaintext secrets surviving on the sandbox filesystem after a failed provision is exactly what + that unlink exists to prevent.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text("{not valid json") + provider = _sql_spy_provider(secrets_path=secrets_path) + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "spawn_failed" + assert secrets_path.exists() # the failed load raised before its own unlink ran. + + asyncio.run(provider.close(work_directory=tmp_path)) + assert not secrets_path.exists() + + +# --- B3/§0.3, p6-review-r1: secrets lifetime and injection through provision() ----------------- + + +def test_secrets_are_loaded_and_the_file_is_gone_before_the_first_process_spawns( + tmp_path: Path, +) -> None: + """t5: pins §0.3's lifetime rule end to end — "the provisioner loads this file into memory at + startup and deletes it immediately after loading, BEFORE ANY CUSTOMER PROCESS STARTS." Records + whether the secrets file still existed at the moment of the FIRST spawn call (build step, + managed engine, or source process — whichever the provisioner reaches first).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + existed_at_first_spawn: list[bool] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + existed_at_first_spawn.append(secrets_path.exists()) + return FakeHandle() + + def sync_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess: + existed_at_first_spawn.append(secrets_path.exists()) + return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") + + provider = _sql_spy_provider(runner=runner, sync_run=sync_run, secrets_path=secrets_path) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert existed_at_first_spawn, "expected at least one spawn/sync_run call" + assert not any(existed_at_first_spawn), "secrets.json must be gone before the FIRST spawn" + assert not secrets_path.exists() + + +def test_secrets_are_re_injected_from_memory_on_reset(tmp_path: Path) -> None: + """t5: §0.3 — "the in-memory map lives for the whole job — `reset` restarts... re-inject from + memory." The file is deleted after the first `provision()`, so a `reset()` that still lands + the secret in the agent's env can only be reading it from the in-memory map, not the disk.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + envs_by_argv0: dict[str, dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + # N10, p6-review-r2: no `/work/job.json` exists in this fixture's `tmp_path` — the + # constructor override stands in for it, the same way a local/test lane would. + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, + ) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert not secrets_path.exists() # gone after the first provision() — nothing left to re-read. + + envs_by_argv0.clear() + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + agent_envs = [env for argv, env in envs_by_argv0.items() if argv == ("python3", "agent.py")] + assert agent_envs, "expected the agent process to be respawned by reset()" + assert agent_envs[0].get("LIVEKIT_API_KEY") == "abc123" + + +def test_secret_injection_end_to_end_through_provision(tmp_path: Path) -> None: + """t6: no prior test covered secret injection through `provision()` at all — `select_process_ + secrets` was only ever tested in isolation. A real (on-disk) secrets.json, loaded by a real + `provision()` call, must land the claimed alias in the claiming process's env and must NOT + land in a process that never claimed `target_provider`.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123"})) + envs_by_argv0: dict[tuple[str, ...], dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, # N10, p6-review-r2. + ) + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + agent_env = envs_by_argv0[("python3", "agent.py")] # claims `target_provider` in `_manifest()`. + assert agent_env.get("LIVEKIT_API_KEY") == "abc123" + tools_env = envs_by_argv0[("node", "server.js")] # claims no purpose in `_manifest()`. + assert "LIVEKIT_API_KEY" not in tools_env + + +# --- N10, p6-review-r2 (MAJOR): secret purposes come from the job, not invented ------------------ + + +def test_secrets_with_no_matching_job_purpose_are_dropped_and_logged( + tmp_path: Path, caplog: pytest.LogCaptureFixture, +) -> None: + """N10, p6-review-r2: every alias used to be relabelled `target_provider` unconditionally, + which silently defeats `select_process_secrets`'s own `SOURCE_CHECKOUT` exclusion (F13) the + moment such an alias ever reaches `secrets.json`. An alias with no matching `job.json`/ + `secret_purpose_map` entry must be DROPPED (never injected under a guessed purpose), not + silently relabelled — and the drop must be logged so a real injection gap is diagnosable.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text(json.dumps({"LIVEKIT_API_KEY": "abc123", "UNCLAIMED_ALIAS": "xyz"})) + envs_by_argv0: dict[tuple[str, ...], dict[str, str]] = {} + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + envs_by_argv0[tuple(argv)] = env + return FakeHandle() + + provider = _sql_spy_provider( + runner=runner, secrets_path=secrets_path, + secret_purpose_map={"LIVEKIT_API_KEY": "target_provider"}, # UNCLAIMED_ALIAS has no entry. + ) + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + agent_env = envs_by_argv0[("python3", "agent.py")] + assert agent_env.get("LIVEKIT_API_KEY") == "abc123" # the matched alias still lands. + assert "UNCLAIMED_ALIAS" not in agent_env # the unmatched one is never injected anywhere. + assert any("UNCLAIMED_ALIAS" in record.message for record in caplog.records) + + +def test_read_job_secret_purposes_reads_agent_secret_refs_from_job_json(tmp_path: Path) -> None: + """N10, p6-review-r2: the REAL hosted-path source — `/work/job.json`'s own `agent.secret_refs` + (§1: `{alias: {manager, key, version, purpose}}`) — not just the constructor override.""" + (tmp_path / "job.json").write_text(json.dumps({ + "agent": {"secret_refs": { + "LIVEKIT_API_KEY": { + "manager": "platform-vault", "key": "k", "version": None, + "purpose": "target_provider", + }, + }}, + })) + purposes = pr._read_job_secret_purposes(tmp_path) + assert purposes == {"LIVEKIT_API_KEY": "target_provider"} + + +def test_read_job_secret_purposes_absent_file_returns_empty_and_logs( + tmp_path: Path, caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + purposes = pr._read_job_secret_purposes(tmp_path) + assert purposes == {} + assert any("job.json" in record.message for record in caplog.records) + + +def test_load_and_delete_secrets_malformed_job_json_is_typed_not_attribute_error( + tmp_path: Path, +) -> None: + """N10, p6-review-r2: `read_text`/`json.loads` failures and a non-dict payload used to escape + `provision()` untyped (N9's own class of gap) — a non-dict `job.json` used to raise + `AttributeError` from `raw.items()` deep inside, not the typed failure this pins.""" + (tmp_path / "job.json").write_text(json.dumps(["not", "a", "dict"])) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._read_job_secret_purposes(tmp_path) + assert excinfo.value.code == "spawn_failed" + + +def test_load_and_delete_secrets_malformed_secrets_json_is_typed(tmp_path: Path) -> None: + secrets_path = tmp_path / "secrets.json" + secrets_path.write_text("not valid json{{{") + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr._load_and_delete_secrets(secrets_path, work_directory=tmp_path) + assert excinfo.value.code == "spawn_failed" + + +# --- N2, p6-review-r2 (BLOCKER): promote-path polling --------------------------------------------- + + +def _manifest_with_fast_readiness_poll(*, timeout_seconds: float = 0.3) -> EnvironmentBundleV2: + """A tiny declared readiness timeout/interval so the poll tests below run in milliseconds, not + up to `_DEFAULT_STORE_READY_TIMEOUT_SECONDS`'s 30s floor.""" + return _manifest( + lambda body: {**body, "readiness": [ + {**body["readiness"][0], "timeout_seconds": timeout_seconds, "interval_seconds": 0.01}, + ]} + ) + + +def _recording_selective_prober(*, protocol: CapabilityProtocol, fail_times: int) -> Any: + """Like `_recording_prober`, but only ever fails for calls matching `protocol` — every other + protocol passes immediately. Isolates which call site actually needed to retry, since a bare + `_recording_prober` shared across the whole `SpawnContext` would also be consumed by `_wait_ + for_store_ready`'s own postgres readiness check at freeze time.""" + calls: list[dict[str, Any]] = [] + remaining = [fail_times] + + def prober(**kwargs: Any) -> bool: + calls.append(kwargs) + if kwargs.get("protocol") is protocol and remaining[0] > 0: + remaining[0] -= 1 + return False + return True + + prober.calls = calls # type: ignore[attr-defined] + return prober + + +def test_provision_promotion_polls_the_declared_probe_rather_than_sampling_once( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): `provision()`'s PREPARING->READY promotion used to call + `probe_runtime_health` exactly once, immediately, with no wait — nothing waits for the DAG's + TERMINAL process (`spawn_world` only waits on `depends_on` edges), so a bundle whose + readiness-bearing process is terminal was marked `UNHEALTHY` on every provision regardless of + how quickly it actually came up. A prober that fails twice before passing proves the + promotion actually POLLS. + + Q3, p6-review-r3: `_manifest_with_terminal_readiness`, not `_manifest_with_fast_readiness_ + poll` — the latter kept the `tools` readiness entry, backed by `tools-api`, which `agent`'s + own `depends_on` wait also polls during `spawn_world` and silently ate the scripted failures + BEFORE the promote-time poll ever ran, so `len(http_calls) >= 3` passed even with a reverted, + single-sample `_poll_runtime_health` (verified empirically). The terminal-readiness manifest's + only probe is backed by `agent` itself, which nothing `depends_on` — every HTTP call recorded + is unambiguously the promote-site poll.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.3) + source, bundle_dir = _provision_dirs(tmp_path) + prober = _recording_selective_prober(protocol=CapabilityProtocol.HTTP, fail_times=2) + provider = _sql_spy_provider(prober=prober, secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.READY + http_calls = [c for c in prober.calls if c.get("protocol") is CapabilityProtocol.HTTP] + assert len(http_calls) >= 3 # 2 failures + the passing call. + + +def test_reset_promotion_polls_the_declared_probe_rather_than_sampling_once( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): same fix, the OTHER promote site — `reset()`'s post-sentinel + probe. `template_database` postgres skips `_wait_for_store_ready` entirely on reset (the + job-shared engine is already running). + + Q3, p6-review-r3: `_manifest_with_terminal_readiness`, same reasoning as the provision-side + test above — its only readiness probe is backed by `agent`, the DAG's terminal process, which + nothing `depends_on`, so nothing else on the reset path can consume the flaky prober's failure + budget before the promotion poll itself runs.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.3) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + flaky = _recording_selective_prober(protocol=CapabilityProtocol.HTTP, fail_times=2) + provider._context = dc_replace(provider._context, prober=flaky) + + asyncio.run(provider.reset(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.READY + http_calls = [c for c in flaky.calls if c.get("protocol") is CapabilityProtocol.HTTP] + assert len(http_calls) >= 3 + + +def _manifest_with_terminal_readiness(*, timeout_seconds: float = 0.05) -> EnvironmentBundleV2: + """§2a's own documented shape: a readiness-bearing capability backed by the DAG's TERMINAL + process (`agent` — nothing in `_manifest()`'s topology ever names it in a `depends_on`). + `wait_for_dependency` never checks it during build; only `probe_runtime_health`'s promote-time + sweep of `manifest.readiness` does. The pre-existing `tools` entry is dropped so `tools-api`'s + own depends_on wait (from `agent`) has nothing to poll and returns immediately — build must + succeed regardless of what the promote-time prober does.""" + return _manifest( + lambda body: {**body, "readiness": [ + {"capability": "control", "path": "/health", "timeout_seconds": timeout_seconds, + "interval_seconds": 0.01}, + ], "capabilities": { + **body["capabilities"], + "control": {"protocol": "http", "service": "agent", "configuration_name": None}, + }} + ) + + +def test_provision_promotion_exhausts_the_poll_and_returns_unhealthy_never_raises( + tmp_path: Path, +) -> None: + """N2, p6-review-r2 (BLOCKER): exhaustion is a state DECISION, not a provisioning failure — + `provision()` must return the world `UNHEALTHY`, never raise, when the declared probe never + passes within its own timeout — pinned against the exact §2a "terminal process" shape N2 + itself names.""" + manifest = _manifest_with_terminal_readiness(timeout_seconds=0.05) + source, bundle_dir = _provision_dirs(tmp_path) + + def always_fails_http(**kwargs: Any) -> bool: + return kwargs.get("protocol") is not CapabilityProtocol.HTTP + + provider = _sql_spy_provider( + prober=always_fails_http, secrets_path=tmp_path / "secrets.json", + ) + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.UNHEALTHY + + +# --- N3, p6-review-r2 (MAJOR): a failed first provision() must not poison the provider ----------- + + +def test_provision_failed_first_freeze_resets_identity_so_a_retry_retries_the_build( + tmp_path: Path, +) -> None: + """N3, p6-review-r2: the job identity (`_manifest`/`_bundle_digest`) used to be committed + BEFORE `freeze_baseline` ran — a failure there left the instance claiming an identity with no + build output, so an in-process retry with the SAME bundle took the RECONCILE branch and + raised `internal_invariant_violated` instead of retrying the build and surfacing the real, + often-retryable cause. A `sql_runner` that raises only on its FIRST call (freeze's own `CREATE + DATABASE` for the template) then behaves normally proves the retry re-attempts the build.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + call_count = [0] + real_spy = SqlSpy() + + def flaky_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + call_count[0] += 1 + if call_count[0] == 1: + raise RuntimeError("connection reset by peer") + return real_spy(**kwargs) + + provider = _sql_spy_provider( + sql_runner=flaky_sql_runner, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "store_statement_failed" # the REAL cause, not a bogus invariant. + assert provider._manifest is None # identity reset, not left claiming a half-built job. + + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert runtimes[0].state is pr.RuntimeState.READY + + +# --- N4, p6-review-r2 (MAJOR): no spawn path may orphan a live engine on a partial failure -------- + + +def _manifest_with_two_postgres_stores() -> EnvironmentBundleV2: + return _manifest( + lambda body: {**body, "processes": [ + body["processes"][0], + {"name": "postgres2", "kind": "managed", "engine": "postgres", "version": "16", + "user": "svc-data", "depends_on": []}, + *body["processes"][1:], + ], "capabilities": { + **body["capabilities"], + "database2": { + "protocol": "postgres", "service": "postgres2", + "configuration_name": "DATABASE2_URL", + }, + }, "seed": {"stores": [ + body["seed"]["stores"][0], + {"capability": "database2", "migrations": [], "seed_files": [], + "baseline": {"strategy": "template_database", "inputs_digest": "sha256:" + "e" * 64}, + "sentinel": {"query": "SELECT 1", "expected": "1"}}, + ]}} + ) + + +def test_freeze_baseline_terminates_an_earlier_stores_handle_on_a_later_stores_failure( + tmp_path: Path, +) -> None: + """N4, p6-review-r2 (MAJOR): a raise partway through `freeze_baseline`'s per-store loop used + to leave every EARLIER store's own job-shared engine live and referenced NOWHERE — `self. + _job_shared_handles` is only ever assigned once this function RETURNS. Store 1 (`postgres`) + succeeds and is sealed `template_database`; store 2 (`postgres2`) fails its own seed — store + 1's handle must be terminated, not orphaned holding its formula port. + """ + manifest = _manifest_with_two_postgres_stores() + real_spy = SqlSpy() + + def flaky_sql_runner(**kwargs: Any) -> list[tuple[Any, ...]]: + statement = kwargs["statement"] + if statement.startswith("CREATE DATABASE") and "alk_baseline_postgres2" in statement: + raise RuntimeError("simulated store-2 seed failure") + return real_spy(**kwargs) + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, sql_runner=flaky_sql_runner, work_directory=tmp_path, + ) + handles: list[Any] = [] + original_runner = ctx.runner + + def recording_runner(*args: Any, **kwargs: Any) -> Any: + handle = original_runner(*args, **kwargs) + handles.append(handle) + return handle + + ctx = dc_replace(ctx, runner=recording_runner) + + with pytest.raises(pr.ProcessRuntimeError): + pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + + assert len(handles) == 2 # both postgres/postgres2 were actually spawned. + assert all(handle.terminated for handle in handles) # neither orphaned. + + +def test_ensure_world_publishes_partial_handles_so_close_can_terminate_them( + tmp_path: Path, +) -> None: + """N4, p6-review-r2 (MAJOR): a mid-clone failure (here: `agent`'s own `depends_on` wait on + `tools-api` timing out) used to drop `tools-api`'s already-spawned handle on the floor — `self. + _world_handles[world_index]` was only ever assigned on `_ensure_world`'s SUCCESS path. + `close()` (or the next reconcile) must be able to terminate it instead of orphaning it. + """ + manifest = _manifest_with_fast_readiness_poll(timeout_seconds=0.05) + source, bundle_dir = _provision_dirs(tmp_path) + + def failing_http_prober(**kwargs: Any) -> bool: + return kwargs.get("protocol") is not CapabilityProtocol.HTTP + + provider = _sql_spy_provider( + prober=failing_http_prober, secrets_path=tmp_path / "secrets.json", + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + assert excinfo.value.code == "depends_on_timeout" + tools_handle = provider._world_handles[0]["tools-api"] + assert tools_handle.handle.terminated is False # not yet — close() has not run. + + asyncio.run(provider.close(work_directory=tmp_path)) + assert tools_handle.handle.terminated is True # N4: reachable, and now cleaned up. + + +# --- N9, p6-review-r2 (MAJOR): filesystem failures must not escape untyped ----------------------- + + +def test_spawn_managed_process_data_dir_setup_failure_is_typed_spawn_failed(tmp_path: Path) -> None: + """N9, p6-review-r2: §4.6 — filesystem failures during provisioning are `infrastructure`. + mkdir/chmod/chown for a (re)spawned engine's own data directory used to raise bare, giving + `provision()`'s caller nothing to map.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + + def failing_chown(path: Path, uid: int, gid: int) -> None: + raise PermissionError("simulated: svc-control cannot chown this path") + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + pr.spawn_managed_process( + postgres, port=14000, data_dir=tmp_path / "pg", credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=failing_chown, + ) + assert excinfo.value.code == "spawn_failed" + assert excinfo.value.stage == "spawn" + + +def test_run_conformance_gate_never_raises_on_a_filesystem_fault_during_reset( + tmp_path: Path, +) -> None: + """N9, p6-review-r2: M3's own "TRULY never raises" promise used to hold only for + `ProcessRuntimeError` — `reset_world` beneath the gate runs `rmtree`/`copytree` (`_seal_world_ + store`'s own `datadir_copy` work), and a filesystem fault there must still degrade the gate, + never crash the job.""" + manifest = _manifest( + lambda body: {**body, "seed": {"stores": [{ + **body["seed"]["stores"][0], + "baseline": {"strategy": "datadir_copy", "inputs_digest": "sha256:" + "a" * 64}, + }]}} + ) + sql_spy = SqlSpy() + ctx = _spawn_context(manifest, bundle_dir=tmp_path, sql_runner=sql_spy, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + world_handles = { + index: pr._clone_or_reset_world( + manifest, index, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ).handles + for index in (0, 1) + } + + def failing_copy(src: Path, dst: Path) -> None: + raise OSError("simulated: disk full mid-copy") + + broken_ctx = dc_replace(ctx, copy=failing_copy) + passed, reason = pr.run_conformance_gate( + manifest, context=broken_ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, world_handles=world_handles, + ) + assert (passed, reason) == (False, "conformance_gate_failed") + + +# --- N5/N6, p6-review-r2 (MAJOR): rabbitmq.conf carries credentials; env var matches on disk ------ + + +def _rabbitmq_process() -> Any: + return pr.ManagedProcess.model_validate({ + "name": "queue", "kind": "managed", "engine": "rabbitmq", "version": "3.13", + "user": "svc-data", "depends_on": [], + }) + + +def test_spawn_managed_process_rabbitmq_conf_carries_credentials( + tmp_path: Path, +) -> None: + """N5, p6-review-r2 (MAJOR): a BARE `rabbitmq-server` (no Docker, §0) reads `default_user`/ + `default_pass` from the generated `rabbitmq.conf`, not from `RABBITMQ_DEFAULT_USER`/ + `RABBITMQ_DEFAULT_PASS` (a Docker-entrypoint-only convention) — without this the node + initializes with the built-in `guest` account and every rabbitmq call this module makes + (harness-authenticated) 401s. Q4, p6-review-r3: no `loopback_users` assertion — that line + widened `guest`'s reach instead of narrowing it and was dropped from the generated conf.""" + creds = pr.EngineCredentials(username="harness", password="s3cr3t") + data_dir = tmp_path / "queue" + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + ) + conf_path = data_dir / "rabbitmq.conf" + conf_text = conf_path.read_text(encoding="utf-8") + assert "default_user = harness" in conf_text + assert "default_pass = s3cr3t" in conf_text + assert "loopback_users" not in conf_text + assert (conf_path.stat().st_mode & 0o777) == 0o600 # carries the password in cleartext. + + +def test_spawn_managed_process_rabbitmq_config_file_env_matches_the_on_disk_filename_exactly( + tmp_path: Path, +) -> None: + """N6, p6-review-r2 (MAJOR): `RABBITMQ_CONFIG_FILE` must carry the FULL path, extension + included — the pre-3.7 "the server appends .conf itself" behavior this used to depend on is + not how the catalog's 3.13 works, and the failure is silent (falls back to the default + management port, 15672 — inside §2b's own per-world port band).""" + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "queue" + captured_env: dict[str, str] = {} + + def recording_runner(argv, *, cwd, env, log_path, user=None, group=None): + captured_env.update(env) + return FakeHandle() + + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=recording_runner, sync_run=_fake_sync_run, + ) + conf_path = data_dir / "rabbitmq.conf" + assert captured_env["RABBITMQ_CONFIG_FILE"] == str(conf_path) + assert Path(captured_env["RABBITMQ_CONFIG_FILE"]).name == "rabbitmq.conf" + assert conf_path.exists() + + +def test_spawn_managed_process_rabbitmq_conf_overwrites_a_datadir_copy_style_pre_populated_file( + tmp_path: Path, +) -> None: + """N5, p6-review-r2: a `datadir_copy` restore legitimately copies a PRIOR `rabbitmq.conf` in + before this function ever runs (M8's own note) — the write must overwrite it (fresh + credentials/port every spawn), never fail `FileExistsError` the way an `O_EXCL` create would.""" + data_dir = tmp_path / "queue" + data_dir.mkdir(parents=True) + (data_dir / "rabbitmq.conf").write_text("stale content from a copied baseline\n") + creds = pr.EngineCredentials(username="harness", password="fresh-pw") + pr.spawn_managed_process( + _rabbitmq_process(), port=15003, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + ) + conf_text = (data_dir / "rabbitmq.conf").read_text(encoding="utf-8") + assert "fresh-pw" in conf_text + assert "stale content" not in conf_text + + +# --- Q2, p6-review-r3 (MAJOR): rabbitmq's mnesia data path must not depend on the node name ------- + + +def test_rabbitmq_daemon_env_mnesia_dir_is_node_name_free(tmp_path: Path) -> None: + """Q2, p6-review-r3 (MAJOR): the port-derived `RABBITMQ_NODENAME` must not leak into the mnesia + DATA path — `RABBITMQ_MNESIA_DIR` is a fixed path under `data_dir`, independent of port/node + name. R2, p6-review-r4: `RABBITMQ_MNESIA_BASE` stays set alongside it (rabbitmq derives OTHER + paths — e.g. `RABBITMQ_PLUGINS_EXPAND_DIR` — from the base, not the dir), so every path-valued + key the env names, `MNESIA_BASE` included, must still resolve under `data_dir`.""" + creds = pr.EngineCredentials(username="harness", password="pw") + env = pr.rabbitmq_daemon_env(data_dir=tmp_path, port=15003, credentials=creds) + assert env["RABBITMQ_MNESIA_DIR"] == str(tmp_path / "mnesia") + assert "15003" not in env["RABBITMQ_MNESIA_DIR"] # not derived from the port/node name. + for key in ("RABBITMQ_MNESIA_DIR", "RABBITMQ_MNESIA_BASE", "RABBITMQ_LOG_BASE"): + assert Path(env[key]).is_relative_to(tmp_path), f"{key} escapes data_dir: {env[key]}" + + +def test_rabbitmq_daemon_env_names_the_same_relative_mnesia_path_in_every_world( + tmp_path: Path, +) -> None: + """Q2, p6-review-r3 (MAJOR): world 0's bootstrap and world 1's restored instance boot on + DIFFERENT ports (rabbitmq is never job-shared, §2b), yet both name `/ + mnesia` for `RABBITMQ_MNESIA_DIR` — the port/node name no longer changes which relative path + the daemon looks under. This is necessary, not sufficient, for the copied baseline to actually + load past world 0 — see p6-review-r4 R1 (mnesia's on-disk schema is still node-name-bound).""" + manifest = _manifest_with_rabbitmq_store() + envs: list[dict[str, str]] = [] + + def runner(argv: list[str], *, cwd: Path, env: dict, log_path: Path, user=None, group=None): + envs.append(env) + return FakeHandle() + + ctx = _spawn_context(manifest, bundle_dir=tmp_path, runner=runner, work_directory=tmp_path) + freeze_result = pr.freeze_baseline(manifest, bundle_digest=manifest.digest, context=ctx) + bootstrap_env = next(e for e in envs if "RABBITMQ_MNESIA_DIR" in e) + + envs.clear() + pr._clone_or_reset_world( + manifest, 1, context=ctx, baseline=freeze_result.build_output, + job_shared_handles=freeze_result.job_shared_handles, existing_handles={}, + ) + world1_env = next(e for e in envs if "RABBITMQ_MNESIA_DIR" in e) + + bootstrap_data_dir = tmp_path / "managed" / "queue" + world1_data_dir = tmp_path / "worlds" / "w1" / "queue" + assert bootstrap_env["RABBITMQ_MNESIA_DIR"] == str(bootstrap_data_dir / "mnesia") + assert world1_env["RABBITMQ_MNESIA_DIR"] == str(world1_data_dir / "mnesia") + # different ports still mean different node names (epmd registration) — only the DATA path, + # not the identity, had to stop depending on it. + assert world1_env["RABBITMQ_NODENAME"] != bootstrap_env["RABBITMQ_NODENAME"] + + +# --- N7, p6-review-r2 (MAJOR): rabbitmq readiness must probe BOTH listeners ----------------------- + + +def test_wait_for_store_ready_probes_both_amqp_and_management_listeners_for_rabbitmq( + tmp_path: Path, +) -> None: + """N7, p6-review-r2: the AMQP listener comes up during CORE boot; the management plugin's HTTP + listener comes up in the PLUGIN boot step that follows — the same B4 race, unfixed until now + for the one engine whose seed/sentinel/canary statements all travel over the listener that was + never probed.""" + manifest = _manifest_with_rabbitmq_store() + process = next(p for p in manifest.processes if p.name == "queue") + probed: list[tuple[Any, int, str | None]] = [] + + def recording_prober(**kwargs: Any) -> bool: + probed.append((kwargs["protocol"], kwargs["port"], kwargs.get("path"))) + return True + + ctx = _spawn_context( + manifest, bundle_dir=tmp_path, prober=recording_prober, work_directory=tmp_path, + ) + port = ctx.port_plan.port_for("queue", 0) + pr._wait_for_store_ready(manifest, process, port=port, context=ctx) + + seen = {(protocol, p) for protocol, p, _ in probed} + assert (pr.CapabilityProtocol.AMQP, port) in seen + management_port = pr._rabbitmq_management_port(port) + assert (CapabilityProtocol.HTTP, management_port) in seen + http_paths = [path for protocol, _, path in probed if protocol is CapabilityProtocol.HTTP] + assert "/api/overview" in http_paths + + +# --- N11, p6-review-r2 (MAJOR): a dead job-shared engine must be respawned + every world resealed - + + +def test_reset_respawns_a_dead_job_shared_engine_and_reseals_every_tracked_world( + tmp_path: Path, +) -> None: + """N11, p6-review-r2 (MAJOR): a job-shared `template_database` engine (postgres, in + `_manifest()`) that dies mid-job used to be carried forward as if healthy — every subsequent + `reset()`/reconcile then raised `store_statement_failed` against a data directory sitting + right there on disk. Respawned from that surviving data directory, then EVERY world this + provider currently tracks (not just the one being reset) has its logical DB re-sealed from the + template — `reset()` is only called for world 0 here; world 1's own reseal can only have come + from the respawn's own multi-world loop. + """ + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + sql_spy = SqlSpy() + provider = _sql_spy_provider(sql_runner=sql_spy, secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + old_shared = provider._job_shared_handles["postgres"] + old_shared.handle.terminated = True # simulate the shared postgres dying (e.g. OOM-killed). + sql_spy.calls.clear() + + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + + new_shared = provider._job_shared_handles["postgres"] + assert new_shared is not old_shared # respawned, not silently reused dead. + statements = [statement for _, statement in sql_spy.calls] + assert 'CREATE DATABASE "w0" TEMPLATE "alk_baseline_postgres"' in statements + assert 'CREATE DATABASE "w1" TEMPLATE "alk_baseline_postgres"' in statements # never the + # `reset()` TARGET — only reachable via the respawn's own reseal-every-tracked-world loop. + assert runtimes[0].state is pr.RuntimeState.READY + + +def test_dead_job_shared_engine_respawn_failure_is_typed_naming_the_engine(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + old_shared = provider._job_shared_handles["postgres"] + old_shared.handle.terminated = True + + def failing_runner(argv, *, cwd, env, log_path, user=None, group=None): + raise OSError("no such device") + + provider._context = dc_replace(provider._context, runner=failing_runner) + + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.reset(runtimes[0], work_directory=tmp_path)) + assert excinfo.value.code == "store_statement_failed" + assert excinfo.value.process == "postgres" + + +# --- N12/N13, p6-review-r2 (MINOR): termination order, SIGINT for postgres, kill escalation ------- + + +def test_close_terminates_in_reverse_topological_order(tmp_path: Path) -> None: + """N12, p6-review-r2: dependency-first termination (`spawn_world`'s own insertion order) sends + SIGTERM to a per-world engine while its own dependents may still hold open connections, + guaranteeing the full escalation wait every time. Reversed so dependents (`agent`, `tools-api`) + are torn down BEFORE the engine they depend on (`postgres`).""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + terminate_order: list[str] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + handle = FakeHandle() + name = Path(cwd).name + + def recording_terminate(_name: str = name, _handle: FakeHandle = handle) -> None: + terminate_order.append(_name) + _handle.terminated = True + + handle.terminate = recording_terminate # type: ignore[method-assign] + return handle + + provider = _sql_spy_provider(runner=runner, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + terminate_order.clear() + asyncio.run(provider.close(work_directory=tmp_path)) + per_world_order = [name for name in terminate_order if name in ("tools-api", "agent")] + assert per_world_order == ["agent", "tools-api"] + + +def test_close_prefers_sigint_over_sigterm_for_postgres(tmp_path: Path) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + interrupted: list[str] = [] + + def runner(argv, *, cwd, env, log_path, user=None, group=None): + handle = FakeHandle() + name = Path(cwd).name + + def recording_interrupt(_name: str = name, _handle: FakeHandle = handle) -> None: + interrupted.append(_name) + _handle.interrupted = True + _handle.terminated = True + + handle.interrupt = recording_interrupt # type: ignore[method-assign] + return handle + + provider = _sql_spy_provider(runner=runner, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + asyncio.run(provider.close(work_directory=tmp_path)) + assert "postgres" in interrupted # SIGINT (fast shutdown), never SIGTERM, for postgres. + + +def test_terminate_and_wait_escalates_to_kill_when_the_process_ignores_terminate() -> None: + """N13, p6-review-r2: `FakeHandle.wait` used to report the fake as exited the instant + `terminate()` was called, so no test ever reached the `kill()` branch — M7's escalation half + had zero coverage. A handle that ignores `terminate()`/`interrupt()` entirely (`wait()` only + reports exited once `kill()` has actually run) forces the real escalation path.""" + + class StubbornHandle: + def __init__(self) -> None: + self.terminated = False + self.killed = False + + def is_running(self) -> bool: + return not self.killed + + def captured_output(self) -> str: + return "" + + def terminate(self) -> None: + self.terminated = True # acknowledges the signal but does NOT actually exit. + + def interrupt(self) -> None: + self.terminated = True + + def wait(self, timeout: float) -> bool: + return self.killed + + def kill(self) -> None: + self.killed = True + + handle = StubbornHandle() + pr._terminate_and_wait(handle, timeout=0.01) + assert handle.terminated is True + assert handle.killed is True # escalation actually happened. + assert handle.is_running() is False # reaped. + + +def test_terminate_and_wait_logs_when_the_process_survives_even_kill( + caplog: pytest.LogCaptureFixture, +) -> None: + """N13, p6-review-r2: the post-`kill()` `wait()` return used to be discarded — a child that + STILL had not exited after SIGKILL (a wedged kernel wait, nothing left to escalate to) left no + trace anywhere. Now logged.""" + + class UnkillableHandle: + def is_running(self) -> bool: + return True + + def captured_output(self) -> str: + return "" + + def terminate(self) -> None: + pass + + def interrupt(self) -> None: + pass + + def wait(self, timeout: float) -> bool: + return False # never reports exited, even after kill() below. + + def kill(self) -> None: + pass + + with caplog.at_level("WARNING", logger="fi.alk.harness.process_runtime"): + pr._terminate_and_wait(UnkillableHandle(), timeout=0.01) + assert any("did not exit even after kill" in record.message for record in caplog.records) + + +# --- N14, p6-review-r2 (MINOR): chmod before chown ------------------------------------------------- + + +def test_spawn_managed_process_chmods_before_chowning_the_data_dir(tmp_path: Path) -> None: + """N14, p6-review-r2: `chmod` after `chown` cannot succeed once ownership has moved — `svc- + control` still owns `data_dir` at chmod time and it is free; after chown, a non-root `svc- + control` that can chown but not re-chmod a path it no longer owns would raise + `PermissionError`. Pinned by recording the directory's OWN mode at the moment `chown` fires.""" + manifest = _manifest() + postgres = manifest.processes[0] + creds = pr.EngineCredentials(username="harness", password="pw") + data_dir = tmp_path / "pg" + mode_at_chown_time: list[int] = [] + + def recording_chown(path: Path, uid: int, gid: int) -> None: + if path == data_dir: + mode_at_chown_time.append(data_dir.stat().st_mode & 0o777) + + pr.spawn_managed_process( + postgres, port=14000, data_dir=data_dir, credentials=creds, + runner=lambda *a, **k: FakeHandle(), sync_run=_fake_sync_run, + user_resolver=_fake_user_resolver({"svc-data": (2222, 3333)}), + chown=recording_chown, + ) + assert mode_at_chown_time == [0o700] + + +# --- N16, p6-review-r2 (MINOR): terminate backends before dropping a world's shared logical DB ---- + + +def test_drop_world_shared_databases_terminates_backends_before_dropping(tmp_path: Path) -> None: + """N16, p6-review-r2: mirrors `_reset_template_database`'s own sibling call — a lingering + backend connection on the world's logical DB otherwise blocks this DROP exactly the way it + would block a reuse-time one, silently re-opening the space leak m4 closed.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + sql_spy = SqlSpy() + provider = _sql_spy_provider(sql_runner=sql_spy, secrets_path=tmp_path / "secrets.json") + asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + sql_spy.calls.clear() + provider._drop_world_shared_databases(1) + statements = [statement for _, statement in sql_spy.calls] + assert len(statements) == 2 + assert statements[0].startswith("SELECT pg_terminate_backend") + assert statements[1] == 'DROP DATABASE IF EXISTS "w1"' + + +# --- healthy() port method (task 2c) --------------------------------------------------------------- + + +def test_provider_healthy_port_never_promotes_from_preparing_or_unhealthy(tmp_path: Path) -> None: + """Task 2c / p6-review-r2: unlike the module-level `healthy()`, the PORT method must never + promote from ANY state — not even `preparing`, which only `provision()`'s own poll (N2) + promotes out of.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + + runtime.state = pr.RuntimeState.PREPARING + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is True + assert runtime.state is pr.RuntimeState.PREPARING # never promoted, even on a pass. + + runtime.state = pr.RuntimeState.UNHEALTHY + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is True + assert runtime.state is pr.RuntimeState.UNHEALTHY # still never promoted. + + +def test_provider_healthy_port_demotes_ready_to_unhealthy_on_a_failing_probe( + tmp_path: Path, +) -> None: + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + assert runtime.state is pr.RuntimeState.READY + + provider._prober = lambda **kwargs: False + ok = asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert ok is False + assert runtime.state is pr.RuntimeState.UNHEALTHY + + +def test_provider_healthy_port_demotes_both_the_providers_record_and_the_callers_object( + tmp_path: Path, +) -> None: + """After N1 these are the SAME object, but the port method must not silently rely on that — + pinned by checking both references independently.""" + manifest = _manifest() + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=1, + require_declared_user=False, + )) + runtime = runtimes[0] + provider._prober = lambda **kwargs: False + asyncio.run(provider.healthy(runtime, work_directory=tmp_path)) + assert runtime.state is pr.RuntimeState.UNHEALTHY + assert provider._runtimes[0].state is pr.RuntimeState.UNHEALTHY + assert provider._runtimes[0] is runtime # N1: one object per world for the provider's life. + + +def test_provider_healthy_port_raises_typed_before_provision(tmp_path: Path) -> None: + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + fake_runtime = pr.EnvironmentRuntime( + runtime_id="x", world_index=0, bundle_digest="sha256:" + "0" * 64, + state=pr.RuntimeState.PREPARING, + ) + with pytest.raises(pr.ProcessRuntimeError) as excinfo: + asyncio.run(provider.healthy(fake_runtime, work_directory=tmp_path)) + assert excinfo.value.code == "internal_invariant_violated" + + +# --- voice dispatch identity on runtime metadata --------------------------------------------- + + +def test_spawn_source_process_carries_rendered_dispatch_agent_name(tmp_path: Path) -> None: + """The dial identity exists only in the rendered per-world env; the handle must carry the + RESOLVED value (world index substituted), and carry None when the process declares none.""" + build_dir = tmp_path / "build" / "svc" + build_dir.mkdir(parents=True) + plan = _solo_port_plan("svc") + + def fake_runner(argv, *, cwd, env, log_path, user=None, group=None): + return FakeHandle() + + with_name = pr.spawn_source_process( + _source_process(environment={"LIVEKIT_AGENT_NAME": "agent-w{{WORLD_INDEX}}"}), + build_dir=build_dir, world_dir=tmp_path / "worlds" / "w2" / "svc", world_index=2, + port_plan=plan, configuration_addresses={}, secret_values={}, secret_purposes={}, + runner=fake_runner, + ) + assert with_name.dispatch_agent_name == "agent-w2" + + without = pr.spawn_source_process( + _source_process(environment={}), + build_dir=build_dir, world_dir=tmp_path / "worlds" / "w0" / "svc", world_index=0, + port_plan=plan, configuration_addresses={}, secret_values={}, secret_purposes={}, + runner=fake_runner, + ) + assert without.dispatch_agent_name is None + + +def test_dispatch_metadata_sets_the_key_only_for_exactly_one_distinct_name( + caplog: pytest.LogCaptureFixture, +) -> None: + """Zero agents -> absent (the call runner's typed pre-dial failure owns that outcome); + one distinct name (even from several handles) -> set; conflicting names -> absent, and + loudly, because the pre-dial message cannot say WHY the key is missing.""" + + def handle(name: str | None) -> pr.SpawnedWorldProcess: + return pr.SpawnedWorldProcess( + process_name="p", handle=FakeHandle(), port=1, world_index=0, + dispatch_agent_name=name, + ) + + assert pr._dispatch_metadata({}) == {} + assert pr._dispatch_metadata({"a": handle(None)}) == {} + assert pr._dispatch_metadata({"a": handle("agent-w0")}) == { + "livekit_agent_name": "agent-w0" + } + assert pr._dispatch_metadata({"a": handle("agent-w0"), "b": handle("agent-w0")}) == { + "livekit_agent_name": "agent-w0" + } + with caplog.at_level("WARNING"): + assert pr._dispatch_metadata( + {"a": handle("agent-w0"), "b": handle("other")} + ) == {} + assert any("distinct LIVEKIT_AGENT_NAME" in record.message for record in caplog.records) + + +def test_provision_surfaces_dispatch_identity_on_runtime_metadata(tmp_path: Path) -> None: + """End to end through provision(): a bundle whose agent process declares LIVEKIT_AGENT_NAME + lands the per-world RESOLVED value on runtime.metadata, which is exactly where the call + runner reads its dial identity.""" + + def add_dispatch_env(body: dict[str, Any]) -> dict[str, Any]: + for process in body["processes"]: + if process["name"] == "agent": + process["environment"]["LIVEKIT_AGENT_NAME"] = "agent-w{{WORLD_INDEX}}" + return body + + manifest = _manifest(add_dispatch_env) + source, bundle_dir = _provision_dirs(tmp_path) + provider = _sql_spy_provider(secrets_path=tmp_path / "secrets.json") + runtimes = asyncio.run(provider.provision( + manifest, source=source, bundle_dir=bundle_dir, work_directory=tmp_path, instances=2, + require_declared_user=False, + )) + assert [runtime.metadata for runtime in runtimes] == [ + {"livekit_agent_name": "agent-w0"}, + {"livekit_agent_name": "agent-w1"}, + ] diff --git a/tests/harness/test_runner_conventions.py b/tests/harness/test_runner_conventions.py new file mode 100644 index 00000000..f5444e19 --- /dev/null +++ b/tests/harness/test_runner_conventions.py @@ -0,0 +1,96 @@ +"""Pins the runner return conventions in world-handle-interface.md v3.2's "Return conventions" +block: a check returning ``""``/whitespace counts as held, and a bare ``False`` from ``ready()`` +is broken rather than a plain not-ready. The unchanged neighbors are pinned alongside them so a +future edit to either runner cannot drift the other conventions without a test noticing. +""" + +from __future__ import annotations + +import pytest + +from fi.alk.harness.checks import run_check +from fi.alk.harness.folder import _run, check_ready +from fi.alk.harness.scenario import Scenario + +# --- checks.py: run_check's return convention ------------------------------------------------- + + +@pytest.mark.parametrize("returning", ["None", "True", "''", "' '"]) +def test_check_returning_nothing_or_whitespace_holds(returning: str) -> None: + outcome = run_check(f"def check(world, calls):\n return {returning}\n", None, []) + assert outcome.held and not outcome.broken and outcome.said == "" + + +def test_check_returning_a_non_empty_string_does_not_hold() -> None: + outcome = run_check("def check(world, calls):\n return 'no rows'\n", None, []) + assert not outcome.held and not outcome.broken and outcome.said == "no rows" + + +def test_check_returning_bare_false_does_not_hold_with_reason_false() -> None: + """An agent result, matching checks.py's existing convention — not a broken check.""" + outcome = run_check("def check(world, calls):\n return False\n", None, []) + assert not outcome.held and not outcome.broken and outcome.said == "False" + + +@pytest.mark.parametrize("returning", ["0", "1", "42", "[]", "{}", "0.0", "object()"]) +def test_check_returning_any_other_value_is_broken(returning: str) -> None: + """Anything that is not the held set, a non-empty string, or bare False is our own mistake, + not a finding about the agent — a check that returns 0 by accident must not be scored as a + failing sub-goal with an uninterpretable reason.""" + outcome = run_check(f"def check(world, calls):\n return {returning}\n", None, []) + assert not outcome.held and outcome.broken + + +# --- folder.py: ready()'s return convention --------------------------------------------------- + + +@pytest.mark.parametrize("returning", ["None", "True", "''", "' '"]) +def test_ready_returning_nothing_or_whitespace_is_ready(returning: str) -> None: + outcome = _run(f"def ready(world):\n return {returning}\n", "s/ready.py", "ready", None) + assert outcome.ok and not outcome.broken and outcome.said == "" + + +def test_ready_returning_a_non_empty_string_is_not_ready() -> None: + outcome = _run( + "def ready(world):\n return 'orders pending'\n", "s/ready.py", "ready", None + ) + assert not outcome.ok and not outcome.broken and outcome.said == "orders pending" + + +def test_ready_returning_bare_false_is_broken_not_advisory() -> None: + """The behavioral change: ready() cannot name what it wants, so a bare False is our own + mistake, not a precondition failing on the shared sealed baseline.""" + outcome = _run("def ready(world):\n return False\n", "s/ready.py", "ready", None) + assert not outcome.ok and outcome.broken + + +def test_setup_returning_bare_false_stays_advisory_not_broken() -> None: + """The change is scoped to ready() only; setup()'s own convention is untouched.""" + outcome = _run("def setup(world):\n return False\n", "s/setup.py", "setup", None) + assert not outcome.ok and not outcome.broken + + +@pytest.mark.parametrize("returning", ["0", "1", "[]", "{}", "0.0"]) +def test_ready_returning_any_other_value_is_broken(returning: str) -> None: + """ready() cannot turn a non-string, non-False value into a precondition sentence either, so + it gets the same broken verdict bare False does.""" + outcome = _run(f"def ready(world):\n return {returning}\n", "s/ready.py", "ready", None) + assert not outcome.ok and outcome.broken + + +@pytest.mark.parametrize("returning", ["0", "1", "[]", "{}", "0.0"]) +def test_setup_returning_any_other_value_stays_advisory(returning: str) -> None: + """The widened rule is keyed on entry == "ready", not on the value alone — setup() keeps its + own untouched convention for every one of these values too.""" + outcome = _run(f"def setup(world):\n return {returning}\n", "s/setup.py", "setup", None) + assert not outcome.broken and not outcome.ok + + +def test_check_ready_reports_bare_false_as_broken() -> None: + """Pins the production entry point, not just `_run`: `check_ready` is the only caller that + can ever produce entry == "ready", so a test that only calls `_run` directly would not catch + a rename or rewiring that silently reverted this.""" + outcome = check_ready( + Scenario(name="s", ready_code="def ready(world):\n return False\n"), None + ) + assert not outcome.ok and outcome.broken diff --git a/tests/harness/test_scenario_source.py b/tests/harness/test_scenario_source.py new file mode 100644 index 00000000..22b4dc8e --- /dev/null +++ b/tests/harness/test_scenario_source.py @@ -0,0 +1,1415 @@ +"""`scenario_source.py` -- the reader/compiler/wrapper that turns a bundle's own scenario documents +(the on-disk layout `folder.py` documents: `scenarios//scenario.json` + `setup.py` + +`ready.py` + `checks/.py`) into `hosted_scheduler.py`'s `Scenario`/`SubGoal` objects. + +DUPLICATION DISCLOSURE: this file writes its own scenario-folder fixtures (`_write_scenario`) -- +nothing else in the suite writes this layout, so there is nothing to import instead. The +lightweight `WorldPool`/`HostedScheduler` harness below (`_FakeWorld`, `_FakeWorldFactory`, +`_FakeCallRunner`, `_pool`) is adapted from `tests/harness/test_hosted_scheduler.py`'s own fakes of +the same names (not imported across test modules, per the brief -- copied and trimmed to what this +file needs; `_FakeWorld.read_only()` deliberately SHARES its `rows` dict with the writable world, +unlike that file's `InMemoryWorld.read_only()`, because the consumer-proof test below needs a +check to see what setup actually wrote). No `EnvironmentBundleV2` manifest-writing helper (the +`_build_bundle` pattern in `test_process_preflight.py`) is needed anywhere in this file: neither +`load_scenarios` nor `WorldPool`/`HostedScheduler` reads or validates one. + +`asyncio.run` drives every `async def` seam here, matching every other file in this suite (no +pytest-asyncio dependency in this repo). Verified via `pytest tests/harness/test_scenario_source.py`. +""" + +from __future__ import annotations + +import asyncio +import json +import random +import tempfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from unittest import mock + +import pytest + +from fi.alk.harness import scenario_source as ss +from fi.alk.harness.hosted_entrypoint import ScenarioPreallocationError +from fi.alk.harness.hosted_scheduler import Call, CallOutcome, HostedScheduler, WorldPool +from fi.alk.harness.process_runtime import EnvironmentRuntime, RuntimeState + +# ================================================================================================= +# Scenario-folder fixture writer -- `folder.py`'s documented layout, hand-written (never through +# `fi.alk.harness.folder`/`fi.alk.harness.scenario`, matching the module under test). +# ================================================================================================= + + +@dataclass +class _FakeJob: + """Stands in for `HarnessJob` wherever only `.run_id` is read (p13: + `BundleScenarioSource.build` uses it as the platform-facing provision run name) -- avoids + importing the real pydantic model into this module purely for one attribute.""" + + run_id: str = "job-1" + + +def _write_scenario( + scenarios_root: Path, + name: str, + *, + scenario_key: str = "", + scenario_id: str = "", + sub_goals: list[str] | None = None, + setup_code: str = "", + ready_code: str = "", + checks: dict[str, str] | None = None, + raw_body: dict[str, Any] | None = None, + write_body: bool = True, +) -> Path: + """One scenario folder, written by hand -- deliberately not through pr63's `Scenario` model + (which backfills an empty `scenario_key` via a validator, so a model instance can never + produce the empty-key fixture the brief requires).""" + folder = scenarios_root / name + (folder / "checks").mkdir(parents=True, exist_ok=True) + if write_body: + body = ( + raw_body + if raw_body is not None + else { + "name": name, + "scenario_key": scenario_key, + "scenario_id": scenario_id, + "sub_goals": sub_goals or [], + } + ) + (folder / "scenario.json").write_text(json.dumps(body), encoding="utf-8") + if setup_code: + (folder / "setup.py").write_text(setup_code, encoding="utf-8") + if ready_code: + (folder / "ready.py").write_text(ready_code, encoding="utf-8") + for goal_name, code in (checks or {}).items(): + (folder / "checks" / f"{goal_name}.py").write_text(code, encoding="utf-8") + return folder + + +# ================================================================================================= +# bundle_has_scenarios -- the LAYOUT DECISION's presence test. +# ================================================================================================= + + +def test_bundle_has_scenarios_false_when_no_scenarios_directory(tmp_path: Path) -> None: + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_false_when_scenarios_directory_is_empty(tmp_path: Path) -> None: + (tmp_path / ss.SCENARIOS_DIRNAME).mkdir() + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_false_when_subdirectory_has_no_scenario_json(tmp_path: Path) -> None: + (tmp_path / ss.SCENARIOS_DIRNAME / "s1").mkdir(parents=True) + assert ss.bundle_has_scenarios(tmp_path) is False + + +def test_bundle_has_scenarios_true_with_one_valid_folder(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[]) + assert ss.bundle_has_scenarios(tmp_path) is True + + +# ================================================================================================= +# Reader -- work item 1. +# ================================================================================================= + + +def test_load_scenarios_reads_the_documented_on_disk_layout(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, + "book_a_ride", + scenario_key="book_a_ride", + scenario_id="platform-42", + sub_goals=["created_rider", "judged_tone"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing'\n" + ) + }, + # "judged_tone" deliberately has no checks/ file -- a JUDGED sub-goal. + ) + scenarios = ss.load_scenarios(tmp_path) + assert len(scenarios) == 1 + scenario = scenarios[0] + assert scenario.scenario_key == "book_a_ride" + assert scenario.scenario_id == "platform-42" + assert [g.name for g in scenario.sub_goals] == ["created_rider", "judged_tone"] + deterministic, judged = scenario.sub_goals + assert deterministic.judged == "" + assert judged.judged != "" # mandatory, non-empty marker (CONTRACT QUESTIONS). + + +def test_load_scenarios_sorts_by_folder_name_like_folder_py(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "bravo", scenario_key="bravo") + _write_scenario(root, "alpha", scenario_key="alpha") + scenarios = ss.load_scenarios(tmp_path) + assert [s.scenario_key for s in scenarios] == ["alpha", "bravo"] + + +def test_load_scenarios_missing_setup_or_ready_defaults_to_empty_text(tmp_path: Path) -> None: + # `folder.py`'s `read_folder` treats a missing setup.py/ready.py as "" -- mirrored here. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_load_scenarios_raises_typed_error_for_missing_scenario_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", write_body=False) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_invalid_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = root / "s1" + folder.mkdir(parents=True) + (folder / "scenario.json").write_text("{not valid json", encoding="utf-8") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_non_object_json(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", raw_body=None, write_body=False) + (root / "s1" / "scenario.json").write_text(json.dumps(["not", "an", "object"]), encoding="utf-8") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_for_non_string_sub_goals(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", raw_body={"scenario_key": "s1", "sub_goals": [1, 2]}) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_raises_typed_error_when_scenarios_directory_absent(tmp_path: Path) -> None: + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_load_scenarios_never_silently_skips_a_bad_folder(tmp_path: Path) -> None: + # folder.py's own `read_all` swallows a bad folder and continues -- this module must not: a + # good scenario alongside a broken one still fails the whole load. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "good", scenario_key="good") + _write_scenario(root, "bad", write_body=False) + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# Wrapper field mapping -- work item 3. `scenario_key`/`scenario_id` carried VERBATIM, including +# empty (the empty-key fixture: hand-written, since pr63's model backfills empty keys). +# ================================================================================================= + + +def test_wrapper_carries_empty_scenario_key_and_id_verbatim(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", scenario_id="", sub_goals=[]) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.scenario_key == "" + assert scenario.scenario_id == "" + + +def test_wrapper_preserves_sub_goal_document_order(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["z", "a", "m"]) + scenario = ss.load_scenarios(tmp_path)[0] + assert [g.name for g in scenario.sub_goals] == ["z", "a", "m"] + + +def test_judged_sub_goal_check_returns_none_and_judged_is_non_empty(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["needs_judgment"]) + scenario = ss.load_scenarios(tmp_path)[0] + goal = scenario.sub_goals[0] + assert goal.judged != "" + assert goal.check(object(), []) is None # "held" by the shared return-value convention. + + +def test_deterministic_sub_goal_judged_is_empty(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["holds"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.sub_goals[0].judged == "" + + +# ================================================================================================= +# Compiler -- work item 2. Good/bad, compiled once at load. +# ================================================================================================= + + +def test_compile_empty_source_is_a_no_op_success(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="", ready_code="") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_setup_syntax_error_fails_at_load_as_scenario_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world:\n pass\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_check_syntax_error_fails_at_load_as_scenario_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["broken"], + checks={"broken": "def check(world, calls\n pass\n"}, + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_missing_entry_point_fails_at_load(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="x = 1\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no setup"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-2 (HIGH, p12-review-r1.md) -- an EXISTING but empty/whitespace-only checks/.py must not +# become a vacuously-passing deterministic goal. Absence of the file is what means "judged"; a +# present-but-empty file is malformed, matching `_compile_entry`'s `allow_empty=False` for `check`. +# ================================================================================================= + + +def test_existing_but_empty_check_file_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": ""}) + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +def test_existing_but_whitespace_only_check_file_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": " \n\t\n"} + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +def test_setup_and_ready_still_allow_empty_source_after_the_r1_2_fix(tmp_path: Path) -> None: + # Regression guard: R1-2's `allow_empty=False` is scoped to `check` only -- setup/ready must + # still treat empty/whitespace source as the pre-existing no-op success (`folder.py` parity). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code=" \n", ready_code="") + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None + assert scenario.ready(object()) is None + + +def test_mutation_vacuous_empty_check_pass_is_caught(tmp_path: Path) -> None: + # Mutation table (R1-2): simulates deleting `allow_empty=False` from the `check` call site in + # `_load_one`, i.e. reverting to the pre-fix behavior where an existing-but-empty check file + # silently compiled to a no-op "held" callable -- a vacuous deterministic pass. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["never_checked"], checks={"never_checked": ""}) + + # Baseline: the real fix catches it. + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + def _always_allow_empty(source: str, *, label: str, entry: str, allow_empty: bool = True): + # The mutant: `allow_empty` is accepted but ignored -- `check` is treated exactly like + # `setup`/`ready` again, as if the R1-2 fix's `allow_empty=False` call-site edit were + # reverted. A standalone reimplementation of the pre-fix `_compile_entry` body, not a call + # back into `ss._compile_entry` (which IS this mutant while the patch is active). + del allow_empty + if not source.strip(): + return lambda *args: None + code = compile(source, f"<{label}>", "exec") + namespace: dict[str, Any] = {} + exec(code, namespace) # noqa: S102 + function = namespace.get(entry) + if not callable(function): + raise ss.ScenarioDocumentInvalid(f"{label} defines no {entry}()") + return function + + with mock.patch.object(ss, "_compile_entry", _always_allow_empty): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + goal = scenarios[0].sub_goals[0] + assert goal.judged == "" # still classified deterministic... + assert goal.check(object(), []) is None # ...and the mutant's vacuous "held" verdict + + # Restored: the guard is back. + with pytest.raises(ss.ScenarioDocumentInvalid, match="defines no check"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-3 (MEDIUM, p12-review-r1.md) -- `sub_goals[]` names are used verbatim to build +# `checks/.py`; an absolute or traversal-shaped name must never resolve to a file outside +# `checks/` (bypassing the bundle's own integrity seal) or silently reclassify a deterministic goal +# as judged (a nonexistent path just fails `is_file()`). +# ================================================================================================= + + +def test_absolute_path_subgoal_name_is_rejected(tmp_path: Path) -> None: + outside = tmp_path / "outside.py" + outside.write_text("def check(world, calls):\n return 'should never run'\n", encoding="utf-8") + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[str(outside.with_suffix(""))]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_relative_traversal_subgoal_name_is_rejected(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["../../../../etc/passwd"]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_backslash_subgoal_name_is_rejected(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=["a\\b"]) + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +def test_plain_subgoal_names_are_unaffected_by_the_r1_3_fix(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["holds", "judged_one"], + checks={"holds": "def check(world, calls):\n return None\n"}, + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert [g.name for g in scenario.sub_goals] == ["holds", "judged_one"] + + +def test_mutation_subgoal_name_sanitization_reproduces_the_bundle_escape(tmp_path: Path) -> None: + # Mutation table (R1-3): simulates deleting the `_validate_subgoal_name` call in `_load_one` -- + # an absolute-path sub_goal name would once again resolve `check_path` to a file entirely + # outside `checks/` (and outside the sealed bundle), reading and compiling it. + outside = tmp_path / "outside.py" + outside.write_text("def check(world, calls):\n return 'ran from outside the bundle'\n", encoding="utf-8") + root = tmp_path / ss.SCENARIOS_DIRNAME + escaping_name = str(outside.with_suffix("")) + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[escaping_name]) + + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + with mock.patch.object(ss, "_validate_subgoal_name", lambda name, *, folder_name: None): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + goal = scenarios[0].sub_goals[0] + assert goal.check(object(), []) == "ran from outside the bundle" # read from OUTSIDE checks/ + + with pytest.raises(ss.ScenarioDocumentInvalid, match="not a plain filename"): + ss.load_scenarios(tmp_path) + + +# ================================================================================================= +# R1-1 (CRITICAL, p12-review-r1.md) -- untyped/`BaseException` failures in the load path must never +# bypass the one `except ScenarioDocumentInvalid` clause in `hosted_entrypoint.py::run_job`: a +# scenario document must not be able to set the guest's own exit code (module-level `sys.exit()`) +# or vanish with zero terminal events (an unreadable file, a non-UTF-8 file). Reproduced here at +# the reader/compiler level, directly against `ss.load_scenarios` -- the e2e proof (through the +# REAL `run_job`, asserting the exact exit code and the terminal event) lives in +# `test_hosted_entrypoint.py` for the two triggers that survive `process_preflight.py`'s own +# byte-for-byte digest re-verification (module-level `sys.exit()`, non-UTF-8 bytes -- both hash +# fine as raw bytes). The unreadable-file (chmod 000) trigger is NOT reproduced through the full +# bundle/preflight pipeline: `process_preflight.py::_verify_digest` (a file outside this task's +# four-edit allowlist) re-opens every listed file with a bare `path.open("rb")`, unguarded, and +# would raise its own untyped `PermissionError` before `scenario_source.build()` is ever reached -- +# an orthogonal, pre-existing gap in a module this task cannot touch. Reproduced here instead, +# directly at the boundary this task owns. +# ================================================================================================= + + +def test_setup_module_level_sys_exit_zero_is_typed_document_invalid(tmp_path: Path) -> None: + # The worst case in the finding: `sys.exit(0)` inside scenario code, unguarded, previously + # meant the GUEST process itself exited 0 -- a "clean terminal that never happened" (§0.6). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(0)\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_module_level_sys_exit_three_is_typed_document_invalid(tmp_path: Path) -> None: + # EXIT_FENCED == 3: previously the guest would exit 3, read by the platform as a fenced/ + # superseded attempt rather than a scenario content defect. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(3)\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_check_module_level_bare_sys_exit_is_typed_document_invalid(tmp_path: Path) -> None: + # Bare `sys.exit()` (no argument) is `SystemExit()`, not `SystemExit(int)` -- still a + # `BaseException`, not an `Exception`; a `checks/.py` file is exactly where a generator + # could emit this by omitting the `if __name__ == "__main__":` guard around `folder.py`'s own + # `_RUNNABLE` tail (R1-4). + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", sub_goals=["broken"], + checks={"broken": "import sys\nsys.exit()\n"}, + ) + with pytest.raises(ss.ScenarioDocumentInvalid, match="would not compile"): + ss.load_scenarios(tmp_path) + + +def test_setup_unreadable_file_chmod_000_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world):\n pass\n") + setup_path = folder / "setup.py" + setup_path.chmod(0o000) + try: + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + finally: + setup_path.chmod(0o644) # restore so pytest's tmp_path cleanup can remove it + + +def test_setup_non_utf8_bytes_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = _write_scenario(root, "s1", scenario_key="s1") + (folder / "setup.py").write_bytes(b"def setup(world):\n return '\xff\xfe'\n") + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + + +def test_scenario_json_non_utf8_bytes_is_typed_document_invalid(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + folder = root / "s1" + folder.mkdir(parents=True) + (folder / "scenario.json").write_bytes(b"\xff\xfe not valid utf-8 at all") + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot read"): + ss.load_scenarios(tmp_path) + + +def test_unreadable_scenarios_directory_is_typed_document_invalid_not_an_escape(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + root.mkdir(parents=True) + (root / "s1").mkdir() + (root / "s1" / "scenario.json").write_text(json.dumps({"scenario_key": "s1"}), encoding="utf-8") + root.chmod(0o000) + try: + with pytest.raises(ss.ScenarioDocumentInvalid, match="cannot list"): + ss.load_scenarios(tmp_path) + finally: + root.chmod(0o755) # restore so pytest's tmp_path cleanup can remove it + + +def test_unreadable_scenarios_directory_makes_bundle_has_scenarios_false_not_an_escape( + tmp_path: Path, +) -> None: + # `bundle_has_scenarios` sits OUTSIDE `run_job`'s try/except entirely (R1-1) -- an unreadable + # `scenarios/` must report False (falling back to the safe `NotWiredScenarioSource` default), + # never raise, since there is nothing typed to catch it at that call site. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1") + root.chmod(0o000) + try: + assert ss.bundle_has_scenarios(tmp_path) is False + finally: + root.chmod(0o755) + + +def test_mutation_revert_r1_1_containment_reproduces_the_untyped_escapes(tmp_path: Path) -> None: + # Revert-verify-restore: a SCRATCH copy of this module's pre-R1-1-fix content (never a tracked + # file -- this module did not exist as a tracked file before this task either, see the mutation + # section's own DUPLICATION DISCLOSURE below) is imported under a private name and driven + # through the exact same fixtures the tests above use. It reproduces every one of the four + # untyped escapes the fix closes; the real, fixed `ss` module does not. + import importlib.util + import sys as _sys + + prefix_path = Path( + "/private/tmp/claude-501/-Users-khushalsonawat-Desktop-future-agi/" + "12a30b1b-5fe7-4808-ae3f-103ab50c6ebc/scratchpad/p12fix1/scenario_source_prefix.py" + ) + if not prefix_path.is_file(): + pytest.skip("pre-fix scratch copy not present in this environment") + module_name = "_p12_scenario_source_prefix" + spec = importlib.util.spec_from_file_location(module_name, prefix_path) + assert spec is not None and spec.loader is not None + prefix = importlib.util.module_from_spec(spec) + # dataclasses' `from __future__ import annotations` string-annotation resolution looks the + # module up in `sys.modules` by name -- registered (and cleaned up after) purely for that, + # never left behind for anything else to import. + _sys.modules[module_name] = prefix + try: + spec.loader.exec_module(prefix) + + # (a) module-level sys.exit(0) -- pre-fix: raw SystemExit escapes `load_scenarios` itself. + root = tmp_path / "a" / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="import sys\nsys.exit(0)\n") + with pytest.raises(SystemExit): + prefix.load_scenarios(tmp_path / "a") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "a") + + # (b) non-UTF-8 setup.py -- pre-fix: raw UnicodeDecodeError escapes. + root_b = tmp_path / "b" / ss.SCENARIOS_DIRNAME + folder_b = _write_scenario(root_b, "s1", scenario_key="s1") + (folder_b / "setup.py").write_bytes(b"def setup(world):\n return '\xff\xfe'\n") + with pytest.raises(UnicodeDecodeError): + prefix.load_scenarios(tmp_path / "b") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "b") + + # (c) unreadable setup.py (chmod 000) -- pre-fix: raw PermissionError escapes. + root_c = tmp_path / "c" / ss.SCENARIOS_DIRNAME + folder_c = _write_scenario( + root_c, "s1", scenario_key="s1", setup_code="def setup(world):\n pass\n" + ) + setup_path = folder_c / "setup.py" + setup_path.chmod(0o000) + try: + with pytest.raises(PermissionError): + prefix.load_scenarios(tmp_path / "c") + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path / "c") + finally: + setup_path.chmod(0o644) + finally: + del _sys.modules[module_name] + + +# ================================================================================================= +# Security-shaped (documentation, not enforcement) -- work item 5d. The precedent (`folder.py`'s +# `_run`) restricts nothing: a bare `{}` namespace with full default builtins still reaches `os`, +# `open`, `subprocess`. Pinned here rather than assumed. CONTRACT QUESTIONS: scenario code in the +# hosted guest is unsandboxed beyond the sandbox itself -- no sandbox is added by this module. +# ================================================================================================= + + +def test_scenario_code_is_unsandboxed_importing_os_runs_successfully(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", + setup_code=( + "import os\n" + "def setup(world):\n" + " del world\n" + " return os.getcwd() and None\n" + ), + ) + scenario = ss.load_scenarios(tmp_path)[0] + assert scenario.setup(object()) is None # ran to completion -- `import os` was never blocked. + + +# ================================================================================================= +# Consumer proof (work item 5b) -- real documents -> this adapter -> the REAL HostedScheduler, with +# a fake world/call runner. `WorldPool` needs no real `EnvironmentBundleV2`/provisioner fidelity +# for this (`bundle=object()`, matching test_hosted_scheduler.py's own `_pool` helper) -- only +# `provision()`/`reset()`/`healthy()`/`close()` are ever called on it. +# ================================================================================================= + + +class _FakeProvisioner: + name = "fake-process" + + def __init__(self, instances: int) -> None: + self.instances = instances + self.closed = False + self._runtimes = { + i: EnvironmentRuntime( + runtime_id=f"digest:w{i}", world_index=i, bundle_digest="digest", + state=RuntimeState.READY, endpoints={}, + ) + for i in range(instances) + } + + async def provision(self, bundle, *, source, bundle_dir, work_directory, contract=None, instances=1): + del bundle, source, bundle_dir, work_directory, contract + return [self._runtimes[i] for i in range(instances)] + + async def reset(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> None: + del runtime, work_directory + + async def healthy(self, runtime: EnvironmentRuntime, *, work_directory: Path) -> bool: + del runtime, work_directory + return True + + async def close(self, *, work_directory: Path) -> None: + del work_directory + self.closed = True + + +class _FakeWorld: + """Unlike `test_hosted_scheduler.py`'s `InMemoryWorld`, `read_only()` SHARES the `rows` dict + rather than starting a fresh one -- this consumer-proof test needs a `check()` to see what + `setup()` actually wrote, or the deterministic-check assertion below would pass vacuously.""" + + def __init__(self, world_index: int, rng: Any, rows: dict[str, list[dict[str, Any]]] | None = None) -> None: + self.world_index = world_index + self.rng = rng + self.rows: dict[str, list[dict[str, Any]]] = rows if rows is not None else {} + + def state(self, table: str | None = None) -> dict[str, list[dict[str, Any]]]: + return dict(self.rows) if table is None else {table: list(self.rows.get(table, []))} + + def put(self, collection: str, record: dict[str, Any], *, key: str = "") -> dict[str, Any]: + self.rows.setdefault(collection, []).append(record) + return record + + def change(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + del key, changes, by + return 0 + + def drop(self, collection: str, key: str = "", *, by: str = "") -> int: + del key, by + return 0 + + def call(self, name: str, arguments: dict[str, Any] | None = None) -> Call: + raise NotImplementedError + + def query(self, sql: str, params: Any = ()) -> list[dict[str, Any]]: + del sql, params + return [] + + def read_only(self) -> "_FakeWorld": + return _FakeWorld(self.world_index, self.rng, rows=self.rows) + + +class _FakeWorldFactory: + async def create(self, runtime: EnvironmentRuntime, *, rng: random.Random) -> _FakeWorld: + return _FakeWorld(runtime.world_index, rng) + + +class _FakeCallRunner: + def __init__(self, outcomes: dict[str, CallOutcome | Exception]) -> None: + self._outcomes = outcomes + + async def run(self, scenario: Any, runtime: EnvironmentRuntime) -> CallOutcome: + del runtime + outcome = self._outcomes[scenario.scenario_key] + if isinstance(outcome, Exception): + raise outcome + return outcome + + +@dataclass +class _FakeOutbound: + events: list[tuple[str, dict[str, Any]]] = field(default_factory=list) + receipts: list[Any] = field(default_factory=list) + + async def scenario_started(self, *, scenario_key: str, world_index: int, scenario_attempt: int) -> None: + self.events.append(("scenario_started", {"scenario_key": scenario_key})) + + async def scenario_retried(self, *, scenario_key: str, from_world: int, to_world: int) -> None: + self.events.append(("scenario_retried", {})) + + async def world_unhealthy(self, *, world_index: int, cause: str) -> None: + self.events.append(("world_unhealthy", {"cause": cause})) + + async def log(self, *, level: str, message: str) -> None: + self.events.append(("log", {"message": message})) + + async def receipt(self, receipt: Any) -> None: + self.receipts.append(receipt) + + +def _call_outcome() -> CallOutcome: + return CallOutcome( + calls=(Call(name="tool", arguments={}, result="ok", ok=True, error="", refused=False, at=0.0),), + turns=1, started_at="2026-08-25T00:00:00.000Z", ended_at="2026-08-25T00:00:01.000Z", + duration_ms=1000, + ) + + +def _build_fixture_bundle(tmp_path: Path) -> Path: + """Two scenarios: one whose deterministic check genuinely holds against what `setup` wrote, + one whose deterministic check genuinely does not -- proving verdicts are evaluated for real, + not just that a key made it through (the brief's own "key-only assertion cannot detect a + wrapper bug" warning). Each also carries one judged sub-goal.""" + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "passing", + scenario_key="passing", scenario_id="", + sub_goals=["created_rider", "needs_judgment"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ) + }, + ) + _write_scenario( + root, "failing", + scenario_key="failing", scenario_id="", + sub_goals=["created_rider"], + setup_code="def setup(world):\n return None\n", # never creates the rider + ready_code="def ready(world):\n return None\n", + checks={ + "created_rider": ( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ) + }, + ) + return tmp_path + + +def test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() -> None: + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-consumer-")) + bundle_dir = _build_fixture_bundle(tmp_path) + scenarios = ss.load_scenarios(bundle_dir) + assert [s.scenario_key for s in scenarios] == ["failing", "passing"] # sorted by folder name + + outbound = _FakeOutbound() + provisioner = _FakeProvisioner(1) + pool = WorldPool( + provisioner, bundle=object(), source=Path("/work/source"), bundle_dir=bundle_dir, + work_directory=tmp_path, instances=1, outbound=outbound, + ) + await pool.start() + call_runner = _FakeCallRunner( + {"passing": _call_outcome(), "failing": _call_outcome()} + ) + scheduler = HostedScheduler( + pool=pool, world_factory=_FakeWorldFactory(), call_runner=call_runner, + outbound=outbound, job_seed=1, + ) + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=10.0) + await pool.close() + + assert result.aborted is None + receipts_by_key = {r.scenario_key: r for r in result.receipts} + assert set(receipts_by_key) == {"passing", "failing"} + + passing = receipts_by_key["passing"] + assert passing.status == "passed" + goals_by_name = {g.name: g for g in passing.sub_goals} + assert goals_by_name["created_rider"].held is True # a real deterministic check, evaluated + assert goals_by_name["created_rider"].judged is False + assert goals_by_name["needs_judgment"].held is True # placeholder "held" convention + assert goals_by_name["needs_judgment"].judged is True # marks it as not really settled yet + + failing = receipts_by_key["failing"] + assert failing.status == "failed" # the SAME check, genuinely evaluated, genuinely fails + assert failing.sub_goals[0].held is False + assert failing.sub_goals[0].reason == "missing rider" + + asyncio.run(scenario()) + + +def test_runtime_error_inside_setup_reaches_setup_crashed_through_the_real_scheduler() -> None: + # Divergence (a) from folder.py's `_run`: a syntax error is caught at LOAD time. A RUNTIME + # error inside a setup() that compiles fine is a different story -- it must reach + # `hosted_scheduler.py`'s own classification (`_run_phase`'s `_PhaseCrashed` -> + # `setup_crashed`), proven here through the real scheduler, not a fake standing in for it. + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-setup-crash-")) + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "boom", scenario_key="boom", scenario_id="", + sub_goals=[], + setup_code="def setup(world):\n raise ValueError('setup blew up')\n", + ) + scenarios = ss.load_scenarios(tmp_path) + + outbound = _FakeOutbound() + provisioner = _FakeProvisioner(1) + pool = WorldPool( + provisioner, bundle=object(), source=Path("/work/source"), bundle_dir=tmp_path, + work_directory=tmp_path, instances=1, outbound=outbound, + ) + await pool.start() + scheduler = HostedScheduler( + pool=pool, world_factory=_FakeWorldFactory(), call_runner=_FakeCallRunner({}), + outbound=outbound, job_seed=1, + ) + result = await asyncio.wait_for(scheduler.run(scenarios), timeout=10.0) + await pool.close() + + assert len(result.receipts) == 1 + receipt = result.receipts[0] + assert receipt.status == "errored" + assert receipt.failure is not None + assert receipt.failure.code == "setup_crashed" + assert "setup blew up" in receipt.failure.message + + asyncio.run(scenario()) + + +# ================================================================================================= +# R1-4 (MEDIUM, p12-review-r1.md) -- nothing else in this suite feeds the adapter an actual +# `folder.py::write_folder` product; every other fixture here is hand-written. This pins the reader +# against the REAL producer once: a future change to `write_folder`'s layout (e.g. setting +# `namespace["__name__"]` inside `_compile_entry` to make tracebacks readable) would turn every real +# check file's `_RUNNABLE` tail (which every one of them carries) into a module-level `SystemExit` +# at load -- exactly R1-1's worst case -- and an all-hand-written suite would never see it. +# ================================================================================================= + + +def test_real_write_folder_round_trip_matches_the_adapters_reading(tmp_path: Path) -> None: + from fi.alk.harness import folder as fmod + from fi.alk.harness.catalogue import Catalogue, SubGoal + from fi.alk.harness.scenario import Scenario + + catalogue = Catalogue( + sub_goals=[ + SubGoal( + name="created_rider", what="rider row exists", + check=( + "def check(world, calls):\n" + " del calls\n" + " return None if world.state('riders').get('riders') else 'missing rider'\n" + ), + ), + SubGoal(name="polite_tone", what="agent was polite", judged="was the refusal explained?"), + ] + ) + scenario_model = Scenario( + name="book_a_ride", + instruction="book a ride", + sub_goals=["created_rider", "polite_tone"], + setup_code="def setup(world):\n world.put('riders', {'id': 1})\n", + ready_code="def ready(world):\n return None\n", + ) + fmod.write_folder(scenario_model, catalogue, tmp_path) + + # The real producer's own on-disk layout must satisfy `bundle_has_scenarios`'s presence test + # unchanged. + assert ss.bundle_has_scenarios(tmp_path) is True + + scenarios = ss.load_scenarios(tmp_path) + assert len(scenarios) == 1 + scenario = scenarios[0] + goals_by_name = {g.name: g for g in scenario.sub_goals} + assert set(goals_by_name) == {"created_rider", "polite_tone"} + assert goals_by_name["created_rider"].judged == "" # has a real checks/ file + assert goals_by_name["polite_tone"].judged != "" # no checks/ file -- judged, per deterministic() + + world = _FakeWorld(0, random.Random(0)) + assert scenario.setup(world) is None + assert world.rows == {"riders": [{"id": 1}]} + assert scenario.ready(world) is None + # The real producer appends `_RUNNABLE` (ending `if __name__ == "__main__": ... raise + # SystemExit(...)`) to every checks/ file it writes -- this proves `exec(code, {})` resolves + # `__name__` to `'builtins'` (never `'__main__'`), so that tail stays inert, against the REAL + # producer's own bytes rather than a hand-written stand-in that never carries the tail at all. + assert goals_by_name["created_rider"].check(world, []) is None # held, genuinely evaluated + assert goals_by_name["polite_tone"].check(world, []) is None # judged placeholder convention + + +# ================================================================================================= +# R1-5 (MEDIUM, p12-review-r1.md) -- divergence (a) (compile once, at load) moves every scenario +# file's compile+exec OUTSIDE every phase budget the scheduler enforces. A wall-clock budget around +# the load converts a hang into a typed terminal instead of an unbounded one (a worker thread +# cannot actually be canceled, so the background thread is accepted as leaked -- see +# `_LOAD_TIMEOUT_SECONDS`'s docstring). +# ================================================================================================= + + +def test_load_timeout_converts_a_hanging_module_level_scenario_into_a_typed_failure() -> None: + async def scenario() -> None: + tmp_path = Path(tempfile.mkdtemp(prefix="p12-load-timeout-")) + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario( + root, "s1", scenario_key="s1", + # Module-level, not inside setup() -- runs during `_compile_entry`'s `exec`, i.e. + # during the load itself, which is exactly what a real budget must bound. + setup_code="import time\ntime.sleep(1.5)\ndef setup(world):\n pass\n", + ) + source = ss.BundleScenarioSource() + with mock.patch.object(ss, "_LOAD_TIMEOUT_SECONDS", 0.1): + with pytest.raises(ss.ScenarioDocumentInvalid, match="exceeded"): + await source.build( + object(), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) + + asyncio.run(scenario()) + + +def test_load_without_a_hang_is_unaffected_by_the_budget(tmp_path: Path) -> None: + async def scenario() -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", sub_goals=[]) + source = ss.BundleScenarioSource() + + # p13: `build()` now calls `register_with_platform` after load -- this test is about the + # R1-5 timeout BUDGET specifically, not registration, so registration is stubbed to a + # passthrough (registration's own behavior is covered separately, below). + async def _passthrough(scenarios_client, scenarios, *, run_name): + del scenarios_client, run_name + return scenarios + + with mock.patch.object(ss, "register_with_platform", _passthrough): + scenarios = await source.build( + _FakeJob(run_id="job-1"), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) + assert [s.scenario_key for s in scenarios] == ["s1"] + + asyncio.run(scenario()) + + +# ================================================================================================= +# Mutations (work item 6) -- proved via in-memory monkeypatching (`unittest.mock.patch.object`) +# rather than editing bytes on any tracked file on disk: this module (`scenario_source.py`) is a +# brand-new, untracked file at the start of this work, so there is no tracked-file shasum to record +# for these -- the git-recovery safety net the brief's mutation section is guarding against +# (touching a TRACKED file's bytes with no git-based way back) does not apply to a file this task +# itself created. Each mutation patches one function/attribute, runs the assertion that should now +# fail (or the behavior that should now differ), and restores in a `finally` -- `mock.patch.object` +# does this atomically and cannot leave the module in a mutated state even if the assertion raises. +# ================================================================================================= + + +def test_mutation_skip_compile_check_is_killed(tmp_path: Path) -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="s1", setup_code="def setup(world:\n pass\n") + + # Baseline: the real compiler catches the syntax error. + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + # Mutant: a "compiler" that never raises on a bad compile, returning a no-op instead -- + # simulates deleting the try/except around `compile()`/`exec()` in `_compile_entry`. + def _never_fails(source: str, *, label: str, entry: str): + del source, label, entry + return lambda *args: None + + with mock.patch.object(ss, "_compile_entry", _never_fails): + scenarios = ss.load_scenarios(tmp_path) # mutant: no longer raises + assert scenarios[0].setup(object()) is None # confirms the mutant path actually ran + + # Restored: the guard is back. + with pytest.raises(ss.ScenarioDocumentInvalid): + ss.load_scenarios(tmp_path) + + +def test_mutation_judged_flag_dropped_is_killed_by_5b_verdict_assertions() -> None: + # Mutant: `_load_one` always sets `judged=""`, as if the "no checks/.py -> judged" + # branch were deleted. + # + # R1-6/R1-8 fold-in (p12-review-r1.md LOW findings): the original version of this test named + # the 5b consumer-proof test in its own identifier but never actually ran it -- it asserted on + # `_load_one`'s output directly instead, which is true but does not exercise the killing + # assertion it claims to. This version runs the REAL + # `test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios` under the patch and checks + # THAT it raises. No `async def`/`asyncio.run` wrapper needed either (R1-8): that test already + # drives its own `asyncio.run` internally, and nothing here awaits anything. + original_load_one = ss._load_one + + def _dropped_judged(folder: Path): + compiled = original_load_one(folder) + broken_goals = tuple( + ss._CompiledSubGoal(name=g.name, judged="", check=g.check) for g in compiled.sub_goals + ) + return ss._CompiledScenario( + scenario_key=compiled.scenario_key, scenario_id=compiled.scenario_id, + sub_goals=broken_goals, setup=compiled.setup, ready=compiled.ready, + ) + + with mock.patch.object(ss, "_load_one", _dropped_judged): + with pytest.raises(AssertionError): + test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() + + # Restored: the real 5b test passes again. + test_consumer_proof_real_scheduler_evaluates_wrapped_scenarios() + + +def test_mutation_empty_key_reader_synthesizing_a_key_is_caught(tmp_path: Path) -> None: + # Mutant: the reader backfills an empty `scenario_key` with a placeholder instead of carrying + # it verbatim, as pr63's own model validator would. The empty-key fixture's own guarantee + # (verbatim empty carry) must catch this. + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", scenario_id="", sub_goals=[]) + + real = ss.load_scenarios(tmp_path)[0] + assert real.scenario_key == "" # baseline: verbatim carry holds + + original_load_one = ss._load_one + + def _synthesizes_key(folder: Path): + compiled = original_load_one(folder) + key = compiled.scenario_key or "synthesized-key" + return ss._CompiledScenario( + scenario_key=key, scenario_id=compiled.scenario_id, sub_goals=compiled.sub_goals, + setup=compiled.setup, ready=compiled.ready, + ) + + with mock.patch.object(ss, "_load_one", _synthesizes_key): + mutant = ss.load_scenarios(tmp_path)[0] + assert mutant.scenario_key != "" # the mutant's defect: no longer verbatim + + restored = ss.load_scenarios(tmp_path)[0] + assert restored.scenario_key == "" # confirms the patch was fully undone + + +# ================================================================================================= +# p13 -- scenario pre-allocation (`register_with_platform`), wired against the platform's actual +# route (a single `POST .../scenarios/`, discriminated by a body-level `operation` field, keyed +# provision response, full-set `begin`) rather than Karthik's documented two-path/position-ordered +# shape -- see `ScenariosClient`'s and `register_with_platform`'s own docstrings for the file:line +# evidence, and reports/p13-worker-r2.md CONTRACT NOTES for where the two disagree. +# ================================================================================================= + + +def _scenario(key: str, *, scenario_id: str = "") -> ss._CompiledScenario: + return ss._CompiledScenario( + scenario_key=key, scenario_id=scenario_id, sub_goals=(), setup=lambda w: None, + ready=lambda w: None, + ) + + +@dataclass +class _FakeScenariosClient: + """Stands in for `hosted_entrypoint.ScenariosClient` -- records every payload it is called + with (so a test can assert on exactly what `register_with_platform` sends) and returns a + canned `{"result": {...}}`-unwrapped body per call, matching what the real + `ScenariosClient._post` already hands back (the envelope itself is that class's concern, not + this module's -- see `test_scenarios_client_provision_unwraps_the_result_envelope` in + `test_hosted_entrypoint.py`).""" + + provision_response: dict[str, Any] + begin_response: dict[str, Any] = field( + default_factory=lambda: {"test_execution_id": "exec-1", "scenarios": []} + ) + provision_error: Exception | None = None + begin_error: Exception | None = None + provision_calls: list[dict[str, Any]] = field(default_factory=list) + begin_calls: list[dict[str, Any]] = field(default_factory=list) + + def provision(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + del deadline + self.provision_calls.append(payload) + if self.provision_error is not None: + raise self.provision_error + return self.provision_response + + def begin(self, payload: dict[str, Any], *, deadline: float | None = None) -> dict[str, Any]: + del deadline + self.begin_calls.append(payload) + if self.begin_error is not None: + raise self.begin_error + return self.begin_response + + +# ------------------------------------------------------------------------------------------------- +# `BundleScenarioSource.build` wiring -- registration runs after load, before the scheduler; a +# LOCALLY-detectable defect (empty scenario_key) must never reach the network first. +# ------------------------------------------------------------------------------------------------- + + +def test_build_rejects_empty_scenario_key_before_calling_register_with_platform( + tmp_path: Path, +) -> None: + # An empty scenario_key is carried verbatim off the document (this module's own LAYOUT + # DECISION -- never synthesized), but `hosted_entrypoint.py`'s own `_validate_scenarios` would + # reject it downstream as a deterministic, `environment`-domain content defect. Checked here, + # before `register_with_platform` is ever called, so that existing classification wins over a + # round trip that would only rediscover the same defect as a `platform_sync` failure instead. + async def scenario() -> None: + root = tmp_path / ss.SCENARIOS_DIRNAME + _write_scenario(root, "s1", scenario_key="", sub_goals=[]) + source = ss.BundleScenarioSource() + + register_calls: list[Any] = [] + + async def _spy(scenarios_client, scenarios, *, run_name): + register_calls.append((scenarios_client, scenarios, run_name)) + return scenarios + + with mock.patch.object(ss, "register_with_platform", _spy): + with pytest.raises(ss.ScenarioDocumentInvalid, match="scenario_key"): + await source.build( + _FakeJob(), object(), object(), pool=object(), world_factory=object(), + bundle_dir=tmp_path, + ) + assert register_calls == [] # never reached the network + + asyncio.run(scenario()) + + +# ------------------------------------------------------------------------------------------------- +# Payload shape -- quote-driven against Azain's serializers (file:line in each docstring/comment). +# ------------------------------------------------------------------------------------------------- + + +def test_provision_payload_only_sends_fields_azains_serializer_declares() -> None: + # HarnessProvisionPersonaSerializer (futureagi/simulate/serializers/hosted_harness.py:168-174): + # scenario_key is the only REQUIRED field; name/role/situation/outcome/persona are all + # `required=False`. `_CompiledScenario` carries none of the optional ones (this module's own + # LAYOUT DECISION -- see the module docstring), so the payload must send bare scenario_key + # only, never a synthesized value for a field this reader does not have. + scenarios = (_scenario("book_a_ride"), _scenario("cancel_ride")) + payload = ss._provision_payload("run-1", scenarios) + assert payload == { + "operation": "provision", + "name": "run-1", + "personas": [{"scenario_key": "book_a_ride"}, {"scenario_key": "cancel_ride"}], + } + for persona in payload["personas"]: + assert set(persona) == {"scenario_key"} + + +def test_begin_payload_carries_operation_run_test_id_and_the_full_key_set() -> None: + # HarnessScenarioBeginSerializer (futureagi/simulate/serializers/hosted_harness.py:193-198): + # `scenario_keys` is REQUIRED and `allow_empty=False` -- the full sealed set every time, never + # a subset (`begin_scenarios`, services/hosted_harness.py:323-329, 409s on anything less). + scenarios = (_scenario("a"), _scenario("b")) + payload = ss._begin_payload("run-test-1", scenarios) + assert payload == { + "operation": "begin", "run_test_id": "run-test-1", "scenario_keys": ["a", "b"], + } + + +# ------------------------------------------------------------------------------------------------- +# Keyed-response parsing (`_scenario_ids_by_key`) -- dict lookup by scenario_key, never a +# positional zip; every malformed/mismatched shape raises rather than returning a partial mapping. +# ------------------------------------------------------------------------------------------------- + + +def test_scenario_ids_by_key_matches_regardless_of_response_order() -> None: + # The platform's response order is not guaranteed to match submission order -- this is the + # load-bearing case a positional zip would get wrong (mutation-killed below). + submitted = (_scenario("a"), _scenario("b")) + reordered_response = [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ] + assert ss._scenario_ids_by_key(submitted, reordered_response) == { + "a": "platform-a", "b": "platform-b", + } + + +def test_scenario_ids_by_key_raises_typed_error_for_non_list_response() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), {"not": "a list"}) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +def test_scenario_ids_by_key_raises_typed_error_for_a_non_object_entry() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), ["not an object"]) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +def test_scenario_ids_by_key_raises_typed_error_for_unknown_key() -> None: + submitted = (_scenario("a"),) + raw = [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "never-submitted", "scenario_id": "platform-x"}, + ] + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_unknown_key" + + +def test_scenario_ids_by_key_raises_typed_error_for_missing_key() -> None: + submitted = (_scenario("a"), _scenario("b")) + raw = [{"scenario_key": "a", "scenario_id": "platform-a"}] # "b" never comes back + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_missing" + + +def test_scenario_ids_by_key_raises_typed_error_for_duplicate_key() -> None: + submitted = (_scenario("a"),) + raw = [ + {"scenario_key": "a", "scenario_id": "platform-a"}, + {"scenario_key": "a", "scenario_id": "platform-a-again"}, + ] + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, raw) + assert exc_info.value.error.code == "scenario_registration_duplicate_key" + + +def test_scenario_ids_by_key_raises_typed_error_for_empty_scenario_id() -> None: + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key((_scenario("a"),), [{"scenario_key": "a", "scenario_id": ""}]) + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + + +# ------------------------------------------------------------------------------------------------- +# `register_with_platform` -- full sequence: provision -> match (guards) -> begin -> assign. +# ------------------------------------------------------------------------------------------------- + + +def test_register_with_platform_assigns_platform_ids_and_begins_the_full_set() -> None: + async def scenario() -> None: + submitted = (_scenario("a"), _scenario("b")) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + # Deliberately out of order relative to `submitted` -- proves the match is by key. + "scenarios": [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ], + }, + ) + result = await ss.register_with_platform(client, submitted, run_name="run-1") + + assert [s.scenario_key for s in result] == ["a", "b"] # submission order preserved + assert [s.scenario_id for s in result] == ["platform-a", "platform-b"] # matched by key + assert result[0].setup is submitted[0].setup # untouched fields carried through verbatim + assert result[0].ready is submitted[0].ready + assert result[0].sub_goals is submitted[0].sub_goals + + assert client.provision_calls == [ + { + "operation": "provision", "name": "run-1", + "personas": [{"scenario_key": "a"}, {"scenario_key": "b"}], + } + ] + assert client.begin_calls == [ + {"operation": "begin", "run_test_id": "run-test-1", "scenario_keys": ["a", "b"]}, + ] + + asyncio.run(scenario()) + + +def test_register_with_platform_guard_failure_never_calls_begin() -> None: + # "never partial assignment": a guard failure during provision-response parsing must stop + # BEFORE begin is ever called -- no scenario in this batch gets sealed for execution against a + # registration the client-side guard has already rejected. + async def scenario() -> None: + submitted = (_scenario("a"), _scenario("b")) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], # "b" missing + }, + ) + with pytest.raises(ScenarioPreallocationError) as exc_info: + await ss.register_with_platform(client, submitted, run_name="run-1") + assert exc_info.value.error.code == "scenario_registration_missing" + assert client.begin_calls == [] + + asyncio.run(scenario()) + + +def test_register_with_platform_missing_run_test_id_is_a_typed_failure_before_begin() -> None: + async def scenario() -> None: + submitted = (_scenario("a"),) + client = _FakeScenariosClient( + provision_response={ + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], + }, + ) + with pytest.raises(ScenarioPreallocationError) as exc_info: + await ss.register_with_platform(client, submitted, run_name="run-1") + assert exc_info.value.error.code == "scenarios_provision_response_invalid" + assert client.begin_calls == [] + + asyncio.run(scenario()) + + +# ------------------------------------------------------------------------------------------------- +# Mutations (p13 work item 5). +# ------------------------------------------------------------------------------------------------- + + +def test_mutation_positional_zip_matching_is_killed() -> None: + # Mutant: `_scenario_ids_by_key` replaced with a positional zip (Karthik's documented shape -- + # not what the platform actually returns). A reordered response must silently mismatch ids + # under the mutant; the real (key-matching) implementation must not. + submitted = (_scenario("a"), _scenario("b")) + reordered_response = [ + {"scenario_key": "b", "scenario_id": "platform-b"}, + {"scenario_key": "a", "scenario_id": "platform-a"}, + ] + + real = ss._scenario_ids_by_key(submitted, reordered_response) + assert real == {"a": "platform-a", "b": "platform-b"} # baseline: correct regardless of order + + def _positional_zip_mutant(submitted, raw_scenarios): + return { + scenario.scenario_key: entry["scenario_id"] + for scenario, entry in zip(submitted, raw_scenarios, strict=True) + } + + with mock.patch.object(ss, "_scenario_ids_by_key", _positional_zip_mutant): + mutant = ss._scenario_ids_by_key(submitted, reordered_response) + assert mutant == {"a": "platform-b", "b": "platform-a"} # mutant's defect: swapped ids + assert mutant != real + + restored = ss._scenario_ids_by_key(submitted, reordered_response) + assert restored == real # confirms the patch was fully undone + + +def test_mutation_missing_scenario_guard_removed_is_killed() -> None: + # Mutant: the "every submitted key must appear in the response" check deleted from + # `_scenario_ids_by_key` -- as if a job with fewer provisioned scenarios than requested were + # silently accepted instead of failing the whole registration. + submitted = (_scenario("a"), _scenario("b")) + incomplete_response = [{"scenario_key": "a", "scenario_id": "platform-a"}] # "b" never comes back + + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, incomplete_response) # baseline: the real guard catches it + assert exc_info.value.error.code == "scenario_registration_missing" + + def _no_missing_guard_mutant(submitted, raw_scenarios): + del submitted + return {entry["scenario_key"]: entry["scenario_id"] for entry in raw_scenarios} + + with mock.patch.object(ss, "_scenario_ids_by_key", _no_missing_guard_mutant): + mutant = ss._scenario_ids_by_key(submitted, incomplete_response) # mutant: no longer raises + assert "b" not in mutant # confirms the mutant's defect: an incomplete mapping got through + + with pytest.raises(ScenarioPreallocationError) as exc_info: + ss._scenario_ids_by_key(submitted, incomplete_response) # restored + assert exc_info.value.error.code == "scenario_registration_missing" + + +def test_mutation_id_assignment_skipped_is_killed() -> None: + # Mutant: `register_with_platform`'s final `replace(scenario, scenario_id=...)` step deleted -- + # provision/begin both still run (a response-shape bug would not be caught by this mutant), but + # the scenarios handed back to the scheduler never actually carry the platform's id. + async def scenario() -> None: + submitted = (_scenario("a"),) + client = _FakeScenariosClient( + provision_response={ + "run_test_id": "run-test-1", + "scenarios": [{"scenario_key": "a", "scenario_id": "platform-a"}], + }, + ) + + real = await ss.register_with_platform(client, submitted, run_name="run-1") + assert real[0].scenario_id == "platform-a" # baseline: the real fix assigns it + + async def _skip_assignment_mutant(scenarios_client, scenarios, *, run_name): + provision_result = await asyncio.to_thread( + scenarios_client.provision, ss._provision_payload(run_name, scenarios) + ) + await asyncio.to_thread( + scenarios_client.begin, + ss._begin_payload(provision_result["run_test_id"], scenarios), + ) + return scenarios # mutant's defect: returned VERBATIM, ids never merged in + + with mock.patch.object(ss, "register_with_platform", _skip_assignment_mutant): + mutant = await ss.register_with_platform(client, submitted, run_name="run-1") + assert mutant[0].scenario_id == "" # mutant's defect: id never assigned + + restored = await ss.register_with_platform(client, submitted, run_name="run-1") + assert restored[0].scenario_id == "platform-a" # confirms the patch was fully undone + + asyncio.run(scenario()) diff --git a/tests/harness/test_world_handle.py b/tests/harness/test_world_handle.py new file mode 100644 index 00000000..47bc0683 --- /dev/null +++ b/tests/harness/test_world_handle.py @@ -0,0 +1,739 @@ +"""The hosted world handle: `HostedWorld`, its read-only view, and the postgres deltas under it. + +Two lanes, like the store tests this sits beside. Everything that is `HostedWorld`'s own +judgement — the token check, the reserved-name refusal, the row-count cap, the by-resolution, +the read-only view, `call`'s refusal — is proven against a fake store and needs neither Docker +nor a real database nor even psycopg installed. Only "is this really enforced by Postgres, not +just by us" needs the container, and those tests are skipped without one, following the same +rule as the bench Docker lane. +""" + +from __future__ import annotations + +import random +from typing import Any + +import pytest + +from fi.alk.bench._docker import docker_available +from fi.alk.harness.world.errors import ( + WorldError, + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, +) +from fi.alk.harness.world.handle import CONFORMANCE_TABLE, STATE_ROW_CAP, HostedWorld +from fi.alk.harness.world.stores import Snapshot +from fi.alk.harness.world.stores.postgres import PostgresStore + +# --- errors.py: the vocabulary, offline --------------------------------------------------- + + +@pytest.mark.parametrize( + "kind", + [ + WorldQueryRejected, + WorldReadOnly, + WorldReservedName, + WorldStateTooLarge, + WorldUnavailable, + WorldUsageError, + ], +) +def test_every_world_exception_is_a_world_error(kind) -> None: + assert issubclass(kind, WorldError) + assert isinstance(kind("because"), RuntimeError) + + +# --- a fake store: HostedWorld's own judgement, without a database -------------------------- + + +class FakeStore: + """Just enough of `PostgresStore` for `HostedWorld` to run on, without a database. + + `table`/`state` mirror the store's own per-table ordering directly off `primary_keys`, the + way `PostgresStore` resolves it itself; `query` only fakes the three catalogue lookups + `HostedWorld` actually issues (`pg_tables`, `pg_index`, `information_schema.columns`) plus + one quoted-table-name passthrough for `world.query()` itself, since nothing here needs to + parse real SQL to prove what is `HostedWorld`'s own judgement rather than the database's. + """ + + def __init__( + self, + tables: dict[str, list[dict[str, Any]]], + primary_keys: dict[str, list[str]] | None = None, + columns: dict[str, set[str]] | None = None, + ) -> None: + self.tables = {name: list(rows) for name, rows in tables.items()} + self.primary_keys = primary_keys or {} + # A table's columns otherwise come from whatever its rows happen to hold, which is + # nothing for an empty table; an explicit map lets a test give a table its columns + # without needing a seed row just to make a by= resolve, the way it would against a + # real, empty Postgres table. + self.columns = columns or {} + + def _ordered(self, name: str) -> list[dict[str, Any]]: + rows = self.tables.get(name, []) + key = self.primary_keys.get(name) + if key: + return sorted((dict(row) for row in rows), key=lambda row: row[key[0]]) + return [dict(row) for row in rows] + + def state(self, only: list[str] | None = None) -> dict[str, list[dict[str, Any]]]: + names = self.tables if only is None else only + return {name: self._ordered(name) for name in names} + + def table(self, name: str) -> list[dict[str, Any]]: + return self._ordered(name) + + def query(self, statement: str, params: tuple = ()) -> list[dict[str, Any]]: + if "pg_tables" in statement: + return [{"tablename": name} for name in sorted(self.tables)] + if "pg_index" in statement: + table = params[0] + return [{"attname": column} for column in self.primary_keys.get(table, [])] + if "information_schema.columns" in statement: + table = params[0] + known = self.columns.get(table) + if known is None: + known = {column for row in self.tables.get(table, []) for column in row} + return [{"column_name": column} for column in sorted(known)] + for name, rows in self.tables.items(): + if f'"{name}"' in statement: + return [dict(row) for row in rows] + return [] + + def add(self, collection: str, record: dict[str, Any]) -> dict[str, Any]: + stored = dict(record) + stored.setdefault("id", len(self.tables[collection]) + 1) + self.tables[collection].append(stored) + return stored + + def amend(self, collection: str, key: str, changes: dict[str, Any], *, by: str = "") -> int: + changed = 0 + for row in self.tables.get(collection, []): + if str(row.get(by)) == str(key): + row.update(changes) + changed += 1 + return changed + + def remove(self, collection: str, key: str = "", *, by: str = "") -> int: + rows = self.tables.get(collection, []) + if not key: + count = len(rows) + rows.clear() + return count + kept = [row for row in rows if str(row.get(by)) != str(key)] + removed = len(rows) - len(kept) + rows[:] = kept + return removed + + +def _world( + tables: dict[str, list[dict[str, Any]]], + *, + baseline: dict[str, int] | None = None, + primary_keys: dict[str, list[str]] | None = None, + columns: dict[str, set[str]] | None = None, +) -> HostedWorld: + visible = {name: len(rows) for name, rows in tables.items() if name != CONFORMANCE_TABLE} + store = FakeStore(tables, primary_keys, columns) + return HostedWorld( + store, + world_index=3, + rng=random.Random(7), + baseline_row_counts=baseline if baseline is not None else visible, + ) + + +def test_world_index_and_rng_are_carried_through_unchanged() -> None: + rng = random.Random(11) + world = HostedWorld( + FakeStore({"orders": []}), world_index=4, rng=rng, baseline_row_counts={"orders": 0} + ) + assert world.world_index == 4 + assert world.rng is rng + + +# --- state() ----------------------------------------------------------------------------- + + +def test_state_reports_every_visible_table_empty_ones_included() -> None: + world = _world({"customers": [{"id": 1, "name": "ana"}], "orders": []}) + assert world.state() == {"customers": [{"id": 1, "name": "ana"}], "orders": []} + + +def test_state_excludes_the_conformance_table() -> None: + world = _world( + {"orders": [{"id": 1}], CONFORMANCE_TABLE: [{"id": 1, "marker": "alive"}]}, + ) + assert CONFORMANCE_TABLE not in world.state() + + +def test_state_selects_one_table() -> None: + world = _world({"customers": [{"id": 1}], "orders": [{"id": 9}]}) + assert world.state("orders") == {"orders": [{"id": 9}]} + + +def test_state_selector_reflects_the_fakestores_own_ordering() -> None: + """`table()`'s primary-key ordering is `PostgresStore._select_ordered`'s job, proven against + a real database in the docker lane below; this only pins that `HostedWorld` passes through + whatever order `FakeStore.table()` hands back, not that the ordering itself is correct.""" + world = _world( + {"orders": [{"id": 2, "item": "b"}, {"id": 1, "item": "a"}]}, + primary_keys={"orders": ["id"]}, + ) + assert [row["id"] for row in world.state("orders")["orders"]] == [1, 2] + + +def test_state_selector_passes_through_the_fakes_row_order_when_it_has_no_primary_key() -> None: + """Same caveat as above: this is `FakeStore`'s own behaviour when it has no key to sort by, + not a guarantee `HostedWorld` makes about row order.""" + given = [{"item": "b"}, {"item": "a"}] + world = _world({"orders": given}) + assert world.state("orders")["orders"] == given + + +def test_naming_the_conformance_table_is_reserved_not_a_usage_mistake() -> None: + world = _world({"orders": [], CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.state(CONFORMANCE_TABLE) + + +def test_naming_a_table_this_world_does_not_have_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.state("bookings") + + +def test_zero_visible_tables_is_unavailable_not_an_empty_snapshot() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldUnavailable): + world.state() + + +def test_state_selector_on_an_empty_schema_is_unavailable_not_a_usage_error() -> None: + """The empty-schema case is `WorldUnavailable` before the membership test even runs: a world + with nothing in it cannot serve any table name, so blaming the scenario for asking about one + in particular is backwards.""" + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldUnavailable): + world.state("bookings") + + +def test_a_table_over_its_baseline_cap_raises_without_touching_its_rows() -> None: + """The cap is decided from what was measured at freeze, never recounted here. + + The fake table itself holds nothing near 5,000 rows; only the baseline claims it does. If + `state()` re-measured, this would pass instead of raising. + """ + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP + 1}) + with pytest.raises(WorldStateTooLarge): + world.state("orders") + + +def test_a_table_at_or_under_the_cap_is_read_normally() -> None: + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP}) + assert world.state("orders") == {"orders": [{"id": 1}]} + + +def test_bare_state_excludes_an_over_cap_table_without_raising() -> None: + """The exclusion is a property of the bare snapshot, not a failure — one seeded audit table + must not make the primary read verb inert for the whole run. Naming it explicitly still + raises: the exclusion never becomes a way to read the table around its own cap.""" + world = _world( + {"orders": [{"id": 1}], "audit_log": [{"id": 1}]}, + baseline={"orders": 1, "audit_log": STATE_ROW_CAP + 1}, + ) + snapshot = world.state() + assert snapshot == {"orders": [{"id": 1}]} + assert "audit_log" not in snapshot + with pytest.raises(WorldStateTooLarge): + world.state("audit_log") + + +def test_bare_state_raises_when_every_table_is_over_the_cap() -> None: + """Excluding every over-cap table would leave the snapshot `{}` while the schema is not + empty — exactly the vacuous observation `state()` must never produce, since a check reading + an empty snapshot as "nothing there" would pass on a world it never actually looked at — so + the bare call raises instead of quietly handing that back.""" + world = _world( + {"orders": [{"id": 1}], "audit_log": [{"id": 1}]}, + baseline={"orders": STATE_ROW_CAP + 1, "audit_log": STATE_ROW_CAP + 1}, + ) + with pytest.raises(WorldStateTooLarge): + world.state() + + +def test_bare_state_names_both_causes_when_nothing_survives_the_exclusion() -> None: + """The widened message covers the mixed case too: one baseline table over the cap plus one + table the agent created since construction — excluding both would leave the snapshot `{}`, + so the raised message must name both `orders` (over-cap) and `late_table` (never measured), + not just whichever cause the wording happens to lead with.""" + world = _world({"orders": [{"id": 1}]}, baseline={"orders": STATE_ROW_CAP + 1}) + world._store.tables["late_table"] = [{"id": 1}] + with pytest.raises(WorldStateTooLarge) as raised: + world.state() + assert "never measured" in str(raised.value) + assert "late_table" in str(raised.value) and "orders" in str(raised.value) + + +def test_a_table_missing_from_the_baseline_dict_is_unavailable_not_under_cap() -> None: + """A missing entry used to default to a row count of 0 and read as under the cap; the cap + cannot be decided for a table nobody measured at freeze. The check runs once, at + construction, rather than waiting for a scenario's first access to discover the gap.""" + with pytest.raises(WorldUnavailable): + _world({"orders": [{"id": 1}]}, baseline={}) + + +def test_a_table_appearing_after_construction_is_unavailable_on_the_selector_path() -> None: + """`_require_baseline_coverage` only sees the tables visible when the handle was built; a + table that shows up afterward is still absent from `_baseline_row_counts`. Naming it + explicitly must reach the typed `WorldUnavailable` naming the table, not a bare `KeyError` + off an unguarded index — a `WorldUnavailable("")` would pass this test for the wrong reason, + so the message is asserted, not just the type.""" + world = _world({"orders": [{"id": 1}]}) + world._store.tables["late_table"] = [{"id": 1}] + with pytest.raises(WorldUnavailable) as raised: + world.state("late_table") + assert "late_table" in str(raised.value) + + +def test_a_table_appearing_after_construction_is_excluded_from_the_bare_snapshot() -> None: + """Nothing the agent under test does during a call may decide whether bare `state()` raises + — a table it creates since construction is the only way an unmeasured table can exist, so the + bare path excludes it the same way it excludes an over-cap table rather than going + unavailable for the whole snapshot.""" + world = _world({"orders": [{"id": 1}]}) + world._store.tables["late_table"] = [{"id": 1}] + snapshot = world.state() + assert snapshot == {"orders": [{"id": 1}]} + assert "late_table" not in snapshot + + +# --- put / change / drop ------------------------------------------------------------------- + + +def test_put_returns_the_stored_record_key_included() -> None: + world = _world({"customers": []}) + stored = world.put("customers", {"name": "ana"}) + assert stored == {"id": 1, "name": "ana"} + + +def test_put_into_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: []}) + with pytest.raises(WorldReservedName): + world.put(CONFORMANCE_TABLE, {"marker": "x"}) + + +def test_put_into_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"customers": []}) + with pytest.raises(WorldUsageError): + world.put("not_a_table", {"name": "ana"}) + + +def test_put_with_a_key_is_a_usage_error() -> None: + """A hosted table's key is the table's own; passing one is telling it what to call a value + the table is about to generate itself.""" + world = _world({"customers": []}) + with pytest.raises(WorldUsageError): + world.put("customers", {"name": "ana"}, key="c1") + + +def test_change_without_by_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.change("orders", "1", {"item": "b"}) + + +def test_change_with_by_updates_and_reports_the_count() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + assert world.change("orders", "1", {"item": "b"}, by="id") == 1 + + +def test_change_without_by_resolves_a_single_column_primary_key() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}, primary_keys={"orders": ["id"]}) + assert world.change("orders", "1", {"item": "b"}) == 1 + + +def test_change_on_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.change("not_a_table", "1", {"item": "b"}, by="id") + + +def test_change_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.change(CONFORMANCE_TABLE, "1", {"marker": "x"}, by="id") + + +def test_change_with_a_by_that_is_not_a_column_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.change("orders", "1", {"item": "b"}, by="nope") + + +def test_change_by_a_column_works_on_an_empty_table_when_the_fake_declares_its_columns() -> None: + """Proves the columns map on `FakeStore` actually does something: without it an empty + table's columns default to empty and this would raise `WorldUsageError` in the fake even + though a real, empty Postgres table would resolve `by="id"` just fine.""" + world = _world({"orders": []}, columns={"orders": {"id", "item"}}) + assert world.change("orders", "1", {"item": "b"}, by="id") == 0 + + +def test_drop_with_a_key_and_no_by_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldUsageError): + world.drop("orders", "1") + + +def test_drop_with_no_key_needs_no_by() -> None: + world = _world({"orders": [{"id": 1}, {"id": 2}]}) + assert world.drop("orders") == 2 + + +def test_drop_without_by_resolves_a_single_column_primary_key() -> None: + world = _world({"orders": [{"id": 1}, {"id": 2}]}, primary_keys={"orders": ["id"]}) + assert world.drop("orders", "1") == 1 + + +def test_drop_on_something_that_is_not_a_table_is_a_usage_error() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUsageError): + world.drop("not_a_table", "1", by="id") + + +def test_drop_a_reserved_name_is_refused() -> None: + world = _world({CONFORMANCE_TABLE: [{"id": 1}]}) + with pytest.raises(WorldReservedName): + world.drop(CONFORMANCE_TABLE) + + +def test_drop_with_a_by_that_is_not_a_column_is_a_usage_error() -> None: + world = _world({"orders": [{"id": 1, "item": "a"}]}) + with pytest.raises(WorldUsageError): + world.drop("orders", "1", by="nope") + + +# --- query(): the token check ---------------------------------------------------------------- + + +@pytest.mark.parametrize( + "sql", + [ + "SELECT * FROM orders", + "select * from orders", + " -- a leading comment\nSELECT * FROM orders", + "/* block */ SELECT * FROM orders", + "WITH recent AS (SELECT * FROM orders) SELECT * FROM recent", + "VALUES (1), (2)", + "SELECT * FROM orders;", + "SELECT * FROM orders WHERE note = 'ends with a semicolon;'", + "SELECT * FROM orders WHERE note = 'ask for update'", + "SELECT * FROM orders WHERE note = E'a semicolon inside an escape string: ;'", + "SELECT * FROM orders WHERE note = E'it\\'s for update'", + ], +) +def test_query_accepts_one_read_statement(sql) -> None: + """Not rejected, whatever it finds to read - the token check is a shape question only.""" + world = _world({"orders": [{"id": 1}]}) + assert isinstance(world.query(sql), list) + + +@pytest.mark.parametrize( + "sql", + [ + "UPDATE orders SET item = 'x'", + "DELETE FROM orders", + "INSERT INTO orders (item) VALUES ('x')", + "SELECT * FROM orders; DROP TABLE orders", + "SELECT * FROM orders FOR UPDATE", + "SELECT * FROM orders FOR SHARE", + " ", + ], +) +def test_query_rejects_anything_that_is_not_one_plain_read(sql) -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query(sql) + + +def test_query_accepts_a_semicolon_inside_a_dollar_quoted_literal() -> None: + """A dollar-quoted span is a literal like any other quoted one; a `;` inside `$$...$$` must + not count toward the one-statement rule any more than one inside `'...'` does.""" + world = _world({"orders": [{"id": 1}]}) + assert isinstance(world.query("SELECT $$contains a semicolon: ;$$ AS note"), list) + + +def test_query_rejects_a_second_statement_hidden_behind_a_dollar_quoted_apostrophe() -> None: + """Before `_blank` knew dollar-quoting, the `'` inside `$$it's fine$$` opened a bogus quote + span that swallowed everything after it, including the real statement separator, so a + second statement rode through unrejected. Recognising the dollar-quoted span as one unit is + what keeps the semicolon between the two real statements visible.""" + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query("SELECT $$it's fine$$ AS a; SELECT 2") + + +def test_query_rejects_a_second_statement_hidden_behind_an_escaped_quote() -> None: + """Before `_blank` knew `E'...'`'s backslash escapes, the `\\'` inside one closed the + literal early; the real closing quote right after it then read as opening a fresh span, and + everything up to the next quote — semicolon, second statement and all — vanished into it + unrejected.""" + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query("SELECT * FROM orders WHERE note = E'x\\'y' ; DELETE FROM orders") + + +@pytest.mark.parametrize( + "sql", + [ + f"SELECT * FROM {CONFORMANCE_TABLE}", + f'SELECT * FROM "{CONFORMANCE_TABLE}"', + ], +) +def test_query_rejects_naming_the_conformance_table(sql) -> None: + world = _world({"orders": [{"id": 1}]}) + with pytest.raises(WorldQueryRejected): + world.query(sql) + + +# --- read_only(): the view ready() and check() actually get --------------------------------- + + +def test_read_only_carries_world_index_and_rng() -> None: + rng = random.Random(5) + world = HostedWorld( + FakeStore({"orders": []}), world_index=2, rng=rng, baseline_row_counts={"orders": 0} + ) + view = world.read_only() + assert view.world_index == 2 + assert view.rng is rng + + +def test_read_only_still_reads() -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + assert view.state() == {"orders": [{"id": 1}]} + assert view.query('SELECT * FROM "orders"') == [{"id": 1}] + + +@pytest.mark.parametrize( + "act", + [ + lambda view: view.put("orders", {"item": "x"}), + lambda view: view.change("orders", "1", {"item": "x"}, by="id"), + lambda view: view.drop("orders", "1", by="id"), + lambda view: view.call("some_tool"), + ], +) +def test_read_only_refuses_every_write_verb(act) -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + with pytest.raises(WorldReadOnly): + act(view) + + +def test_read_only_hides_unknown_capability_probes_behind_a_plain_attribute_error() -> None: + """`prove.py`/`probe.py`/`run/voice.py` all reach for capability attributes on world objects + through `hasattr`/`getattr(..., default)`; only a plain `AttributeError` makes that pattern + work, so an unknown name here must read as one instead of as a write refusal.""" + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + assert hasattr(view, "forward") is False + assert getattr(view, "runtime_tools", set()) == set() + + +def test_read_only_dunder_lookups_stay_plain_attribute_errors() -> None: + world = _world({"orders": [{"id": 1}]}) + view = world.read_only() + with pytest.raises(AttributeError): + view.__wrapped__ + + +# --- call(): not implemented, the http_tool shim's wire format is unpinned ------------------ + + +def test_call_is_unavailable() -> None: + world = _world({"orders": []}) + with pytest.raises(WorldUnavailable): + world.call("some_tool") + + +# --- with docker: the deltas that only Postgres itself can really prove -------------------- + +pg = pytest.mark.skipif(not docker_available(), reason="docker daemon unavailable") + +SCHEMA = """ +CREATE TABLE _alk_conformance ( + id serial PRIMARY KEY, + marker text NOT NULL +); +CREATE TABLE customers ( + id serial PRIMARY KEY, + name text NOT NULL +); +CREATE TABLE orders ( + id serial PRIMARY KEY, + customer_id int NOT NULL REFERENCES customers(id), + item text NOT NULL +); +""" + +SEED = """ +INSERT INTO _alk_conformance (marker) VALUES ('alive'); +INSERT INTO customers (name) VALUES ('ana'), ('bo'); +INSERT INTO orders (customer_id, item) VALUES (1, 'turkey'); +""" + + +@pytest.fixture(scope="module") +def store(): + running = PostgresStore(version="16") + running.start() + try: + running.apply(SCHEMA) + yield running + finally: + running.stop() + + +@pytest.fixture() +def seeded(store): + store.restore(Snapshot()) + store.apply(SEED) + return store + + +@pg +def test_postgres_query_reads_real_rows(seeded) -> None: + rows = seeded.query('SELECT * FROM "customers" ORDER BY "id"') + assert [row["name"] for row in rows] == ["ana", "bo"] + + +@pg +def test_postgres_query_refuses_a_write_at_the_database(seeded) -> None: + """The friendliness check lives in `HostedWorld`; the store's own guard is this.""" + import psycopg + + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + seeded.query("DELETE FROM orders") + # And the delete really did not happen. + assert seeded.state()["orders"] + + +@pg +def test_postgres_query_accepts_a_percent_literal_with_no_bound_params(seeded) -> None: + """An empty params tuple is still `not None` to psycopg's placeholder scanner, which would + otherwise read the `%t` in `%turkey%` as an unmatched placeholder and raise before the + statement ever reaches Postgres.""" + rows = seeded.query("SELECT * FROM orders WHERE item LIKE '%turkey%'") + assert [row["item"] for row in rows] == ["turkey"] + + +@pg +def test_postgres_add_returns_the_row_with_its_generated_key(seeded) -> None: + stored = seeded.add("customers", {"name": "cy"}) + assert stored["name"] == "cy" + assert isinstance(stored["id"], int) + assert stored in seeded.state()["customers"] + + +@pg +def test_hosted_world_query_refuses_a_data_modifying_cte(seeded) -> None: + """A data-modifying CTE passes the token check (its leading keyword is WITH, a read + keyword); the read-only transaction underneath is what actually stops the INSERT it + hides, exactly as the friendliness check was always meant to be backed up.""" + import psycopg + + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(9), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + with pytest.raises(psycopg.errors.ReadOnlySqlTransaction): + world.query( + "WITH w AS (INSERT INTO orders (customer_id, item) VALUES (1, 'ham') " + "RETURNING *) SELECT * FROM w" + ) + assert len(world.state()["orders"]) == 1 + + +@pg +def test_hosted_world_query_refuses_a_result_with_duplicate_column_labels(seeded) -> None: + """`o.*, c.*` on a join gives both tables' `id` the same label; silently keeping only the + last one under `dict(zip(...))` would let a check read the wrong table's key with no error + anywhere - refusing beats a check that is quietly wrong.""" + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(3), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + with pytest.raises(WorldQueryRejected): + world.query("SELECT o.*, c.* FROM orders o JOIN customers c ON c.id = o.customer_id") + + +@pg +def test_hosted_world_end_to_end_against_a_real_database(seeded) -> None: + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(42), + baseline_row_counts={"customers": 2, "orders": 1}, + ) + + state = world.state() + assert CONFORMANCE_TABLE not in state + assert [row["name"] for row in state["customers"]] == ["ana", "bo"] + + with pytest.raises(WorldReservedName): + world.state(CONFORMANCE_TABLE) + + stored = world.put("customers", {"name": "cy"}) + assert stored["name"] == "cy" + + changed = world.change("orders", "1", {"item": "ham"}, by="id") + assert changed == 1 + assert world.state("orders")["orders"][0]["item"] == "ham" + + dropped = world.drop("orders", "1", by="id") + assert dropped == 1 + assert world.state("orders") == {"orders": []} + + assert world.query("SELECT name FROM customers ORDER BY name") == [ + {"name": "ana"}, + {"name": "bo"}, + {"name": "cy"}, + ] + + +@pg +def test_hosted_world_cap_uses_the_baseline_not_a_live_count(seeded) -> None: + world = HostedWorld( + seeded, + world_index=0, + rng=random.Random(1), + baseline_row_counts={"customers": STATE_ROW_CAP + 1, "orders": 1}, + ) + with pytest.raises(WorldStateTooLarge): + world.state("customers") + # A table the baseline did not flag reads normally in the same call. + assert world.state("orders") == {"orders": [{"id": 1, "customer_id": 1, "item": "turkey"}]} + + +@pg +def test_hosted_world_on_an_empty_schema_is_unavailable() -> None: + empty = PostgresStore(version="16") + empty.start() + try: + world = HostedWorld(empty, world_index=0, rng=random.Random(1), baseline_row_counts={}) + with pytest.raises(WorldUnavailable): + world.state() + finally: + empty.stop() diff --git a/tests/harness/test_world_stores_container.py b/tests/harness/test_world_stores_container.py new file mode 100644 index 00000000..e2a1cb21 --- /dev/null +++ b/tests/harness/test_world_stores_container.py @@ -0,0 +1,52 @@ +"""Regression: `ContainerStore._await_ready`'s timeout path must not leak the container it +started. + +`start()` sets `_started = True` (world/stores/container.py) BEFORE `_await_ready()` runs, so a +container that never answers within `READY_TIMEOUT_SECONDS` used to raise `StoreError` straight +out of `start()` with the container still running -- nothing else ever calls `stop()` on a store +whose own `start()` raised, so the container was orphaned for good. Docker-gated, following the +same rule as `tests/test_harness_stores.py`'s own Postgres lane. +""" + +from __future__ import annotations + +import pytest + +from fi.alk.bench._docker import docker_available +from fi.alk.harness.world.stores import container +from fi.alk.harness.world.stores.postgres import PostgresStore + +pg = pytest.mark.skipif(not docker_available(), reason="docker daemon unavailable") + + +class NeverReadyStore(PostgresStore): + """Boots exactly like `PostgresStore` (so the container genuinely starts and stays running -- + a real "running but not an engine that answers" case, not a `docker run` failure) but its + `probe()` never succeeds, so `_await_ready` always times out.""" + + def probe(self) -> None: + raise ConnectionError("deliberately never ready, for the leak-on-timeout regression") + + +def _container_exists(name: str) -> bool: + listed = container.docker( + "ps", "-a", "--filter", f"name=^{name}$", "--format", "{{.Names}}", check=False + ) + return name in listed.splitlines() + + +@pg +def test_await_ready_timeout_removes_the_container_it_started(monkeypatch) -> None: + # A short deadline keeps this fast -- the bug and the fix are both about WHAT HAPPENS on + # timeout, not about how long a real engine takes to boot. + monkeypatch.setattr(container, "READY_TIMEOUT_SECONDS", 0.5) + store = NeverReadyStore() + try: + with pytest.raises(container.StoreError, match="did not answer"): + store.start() + assert not _container_exists(store.container) + assert store._started is False # same post-teardown state `stop()` leaves on success + assert store.port is None + finally: + # Backstop only -- a passing test already removed it via the fixed timeout path. + store.stop() diff --git a/tests/test_harness.py b/tests/test_harness.py index 02f646e1..ffa4f364 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -5344,9 +5344,10 @@ def test_a_setup_or_ready_that_says_nothing_is_not_a_complaint(): ) assert not complained.ok and complained.said == "no pending orders" - # And False is a failure that says nothing, so the message says that rather than being blank. + # And False from ready() names no precondition, so it is broken rather than a plain failure — + # the message says that rather than being blank. bare = _run("def ready(world):\n return False\n", "s/ready.py", "ready", None) - assert not bare.ok and "without saying what is wrong" in bare.said + assert not bare.ok and bare.broken and "without saying what is wrong" in bare.said def test_an_optional_field_may_be_null():