From 873cecd87b193b69b8a5bb6e1d9a5406b1cf6eb5 Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 08:17:35 +0200 Subject: [PATCH 1/6] Recover EP purge after partial process-crash deletion --- .../ENGINEERING_PLATFORM_ROADMAP.md | 10 + .../EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md | 29 ++ .../system_instance_lifecycle.py | 74 +++-- .../system_instance_provisioner.py | 3 +- .../test_system_instance_provisioner.py | 60 ++++ .../system_instance_purge_sigkill.py | 270 ++++++++++++++++++ 6 files changed, 426 insertions(+), 20 deletions(-) create mode 100644 tools/qualification/system_instance_purge_sigkill.py diff --git a/docs/development/ENGINEERING_PLATFORM_ROADMAP.md b/docs/development/ENGINEERING_PLATFORM_ROADMAP.md index bc039aa8..c67c243f 100644 --- a/docs/development/ENGINEERING_PLATFORM_ROADMAP.md +++ b/docs/development/ENGINEERING_PLATFORM_ROADMAP.md @@ -9,6 +9,16 @@ may extend productive work; reasoning text cannot, and the hard maximum never resets. The Operations Console projects both limits. See [the owning design](../engineering/PROGRESS_AWARE_PROVIDER_DEADLINES_V2.md). +## EP Server system-instance process-crash qualification — 2026-09-29 + +The bounded process-crash subset of the existing OI-2 system-instance +lifecycle is tracked in [EP Server system/multi-instance lifecycle](../engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md#abrupt-process-stop-qualification-revision-26) +and canonical LANE_2 register `pcvantol/forge#142` revision 26. The public +2.3.104 wheel has a reproduced partial-`PURGE` crash gap. Source and locally +built candidate qualification are in progress; this note creates no new node, +does not change broader OI-2 status and claims no published fix or installer +acceptance. + ## Release and operational installation lifecycle V1 — active coordinated plan Repository evidence recorded on 2026-09-09 reads `origin/main` as diff --git a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md index ccc30ff2..acd53b77 100644 --- a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md +++ b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md @@ -292,3 +292,32 @@ tamper rejection, explicit permanent purge, sibling-instance non-interference, lost-response recovery for preserve/restore/purge, receipt replay and existing destructive `remove` compatibility. Qualification uses isolated product fixtures only; no production Server/CENTRAL/credential mutation is authorized. + +### Abrupt process-stop qualification (revision 26) + +The historical 2.3.104 installed-wheel qualification above covered controlled +interruption and lost responses. The separate LANE_2 revision-26 qualification +found a real SIGKILL gap in `PURGE`: the legacy `REMOVE` receipt is published +before recursive deletion. If the process dies after deleting `instance.json` +but before the instance root and lifecycle tombstone are removed, the normal +same-operation `purge` retry cannot reopen the descriptor. The exact published +2.3.104 wheel reproduces this failure under Python 3.14; it remains historical +`GAP_PROVEN` evidence and is not relabeled. + +The bounded recovery correction retains the prepared root's device/inode and +minimal topology outside the mutable instance tree. After a terminal exact +`REMOVE` receipt, recovery admits only that same root, rejects a replacement +root or changed surviving descriptor, checks the remaining tree for unsafe +entries, and finishes the original deletion before publishing the purge +tombstone and lifecycle receipt. The existing lifecycle request and receipt +identities remain unchanged. The reproducible installed-wheel SIGKILL harness +is `tools/qualification/system_instance_purge_sigkill.py`; it reports durable +`PURGE_READY`, physical partial deletion, exact mutator signal exit, surviving +descendants, same-operation recovery, terminal replay and sibling byte identity. + +Source tests and a locally built candidate-wheel SIGKILL run have passed this +specific partial-deletion cell. The new protected source, any required normal +release and requalification against its published bytes are still required +before this process-crash subset may be called qualified. Other revision-26 +PRESERVE/RESTORE/PURGE crash cells remain separately open; no physical reboot, +power-loss, production service or installer acceptance is inferred. diff --git a/src/engineering_platform/system_instance_lifecycle.py b/src/engineering_platform/system_instance_lifecycle.py index 1870a5ab..32b0745e 100644 --- a/src/engineering_platform/system_instance_lifecycle.py +++ b/src/engineering_platform/system_instance_lifecycle.py @@ -477,40 +477,78 @@ def prepare_purge( "PURGE_READY", mutable_instance_data_digest=tree_digest, descriptor_digest=request["descriptor_digest"], + # REMOVE publishes its receipt before deleting this tree. A + # process kill can erase instance.json before rmtree finishes. + # Retain only the topology and root identity needed to resume + # that exact deletion without adopting a replacement root. + display_label=instance.display_label, + service_account=instance.service_account, + root_device=instance.root.stat(follow_symlinks=False).st_dev, + root_inode=instance.root.stat(follow_symlinks=False).st_ino, ) def resume_purge_target( self, - instance: system_installation_topology.SystemInstanceTopology, - descriptor: Mapping[str, object], + instance_id: str, operation_id: str, + remove_receipt: Mapping[str, object], ) -> None: - root = self._operation_root(instance.instance_id, operation_id) + root = self._operation_root(instance_id, operation_id) state = _read_json(root / "state.json") if ( state.get("contract") != CONTRACT or state.get("operation") != "PURGE" or state.get("phase") != "PURGE_READY" - or state.get("instance_id") != instance.instance_id - or state.get("descriptor_digest") != _json_digest(dict(descriptor)) + or state.get("instance_id") != instance_id + or state.get("request") != { + "instance_id": instance_id, + "descriptor_digest": state.get("descriptor_digest"), + } + or state.get("request_digest") != _json_digest(state["request"]) + or remove_receipt.get("operation") != "REMOVE" + or remove_receipt.get("operation_id") != operation_id + or remove_receipt.get("instance_id") != instance_id + or not isinstance(remove_receipt.get("evidence"), Mapping) + or remove_receipt["evidence"].get("removed_descriptor_digest") != state.get("descriptor_digest") ): raise SystemInstanceLifecycleError("EP purge recovery evidence is invalid") - with _Lock(instance.lifecycle_lock): + try: + instance = system_installation_topology.system_instance_topology( + self.product, + instance_id=instance_id, + display_label=state["display_label"], + service_account=state["service_account"], + ) + except (KeyError, TypeError, ValueError) as error: + raise SystemInstanceLifecycleError("EP purge recovery topology is invalid") from error + expected = self.product.system_root / "instances" / instance.instance_id + if ( + instance.root != expected + or instance.root.is_symlink() + or not instance.root.is_relative_to(self.product.system_root / "instances") + ): + raise SystemInstanceLifecycleError("EP purge recovery target is invalid") + + def verify_original_root() -> None: + details = instance.root.stat(follow_symlinks=False) + if (details.st_dev, details.st_ino) != (state.get("root_device"), state.get("root_inode")): + raise SystemInstanceLifecycleError("EP purge recovery target was replaced") + + # Check before opening the instance-owned lock, which may recreate a + # lock directory already erased by the interrupted deletion. + verify_original_root() + with _Lock(root / "purge-recovery.lock"), _Lock(instance.lifecycle_lock): + verify_original_root() self.controller.remove(instance) if self.controller.loaded(instance) or self.service_interpreter(instance) is not None: raise SystemInstanceLifecycleError("EP purge recovery service was not fully removed") - if _tree_digest( - instance.root, - ignored_roots=(instance.operations_root / operation_id,), - ) != state.get("mutable_instance_data_digest"): - raise SystemInstanceLifecycleError("EP purge recovery target changed after preparation") - expected = self.product.system_root / "instances" / instance.instance_id - if ( - instance.root != expected - or instance.root.is_symlink() - or not instance.root.is_relative_to(self.product.system_root / "instances") - ): - raise SystemInstanceLifecycleError("EP purge recovery target is invalid") + # The exact root survives until rmtree's final step. Its contents + # may be a strict subset after SIGKILL, so the original whole-tree + # digest cannot be required here. Reject unsafe remaining entries. + _tree_digest(instance.root, ignored_roots=(instance.operations_root / operation_id,)) + if instance.descriptor.exists(): + if _json_digest(_read_json(instance.descriptor)) != state.get("descriptor_digest"): + raise SystemInstanceLifecycleError("EP purge recovery descriptor changed") shutil.rmtree(instance.root) def record_purge( diff --git a/src/engineering_platform/system_instance_provisioner.py b/src/engineering_platform/system_instance_provisioner.py index c7ef3289..a718aa87 100644 --- a/src/engineering_platform/system_instance_provisioner.py +++ b/src/engineering_platform/system_instance_provisioner.py @@ -665,8 +665,7 @@ def purge(self, instance_id: str, operation_id: str, *, confirm_instance_id: str else: remove_receipt = prior_remove if instance_root.exists() or instance_root.is_symlink(): - instance, descriptor = self._instance_from_descriptor(instance_id) - lifecycle.resume_purge_target(instance, descriptor, operation_id) + lifecycle.resume_purge_target(instance_id, operation_id, remove_receipt) receipt = lifecycle.record_purge(instance_id, operation_id, remove_receipt) return { "contract": system_instance_lifecycle.CONTRACT, diff --git a/tests/engineering/test_system_instance_provisioner.py b/tests/engineering/test_system_instance_provisioner.py index 69a6144b..a2c68231 100644 --- a/tests/engineering/test_system_instance_provisioner.py +++ b/tests/engineering/test_system_instance_provisioner.py @@ -898,6 +898,66 @@ def test_purge_recovers_if_legacy_remove_receipt_precedes_rmtree(self) -> None: purged, ) + def test_purge_recovers_when_process_dies_during_partial_tree_deletion(self) -> None: + self._create("ep-production-0001", "EP production", 8765) + self._create("ep-development-0001", "EP development", 8766) + instance = self._instance("ep-production-0001", "EP production") + sibling = self._instance("ep-development-0001", "EP development") + sibling_before = sibling.descriptor.read_bytes() + (instance.root / "data-sentinel").write_text("erase me", encoding="utf-8") + + def partial_delete(_root: Path) -> None: + instance.descriptor.unlink() + (instance.root / "data-sentinel").unlink() + raise OSError("simulated abrupt stop after partial deletion") + + with patch.object(provisioner_module.shutil, "rmtree", side_effect=partial_delete): + with self.assertRaisesRegex(OSError, "partial deletion"): + self.provisioner.purge( + instance.instance_id, "purge-partial-delete", + confirm_instance_id=instance.instance_id, + ) + self.assertTrue(instance.root.exists()) + self.assertFalse(instance.descriptor.exists()) + self.assertIsNotNone(self.provisioner._terminal_receipt( + instance.instance_id, "purge-partial-delete", "REMOVE", + )) + + receipt = self.provisioner.purge( + instance.instance_id, "purge-partial-delete", + confirm_instance_id=instance.instance_id, + ) + self.assertEqual(receipt["receipt"]["evidence"]["lifecycle_state"], "PURGED") + self.assertFalse(instance.root.exists()) + self.assertEqual(sibling.descriptor.read_bytes(), sibling_before) + self.assertEqual(self.provisioner.purge( + instance.instance_id, "purge-partial-delete", + confirm_instance_id=instance.instance_id, + ), receipt) + + def test_purge_recovery_rejects_replaced_instance_root(self) -> None: + self._create("ep-production-0001", "EP production", 8765) + instance = self._instance("ep-production-0001", "EP production") + with patch.object(provisioner_module.shutil, "rmtree", side_effect=OSError("stop before deletion")): + with self.assertRaisesRegex(OSError, "stop before deletion"): + self.provisioner.purge( + instance.instance_id, "purge-replaced-root", + confirm_instance_id=instance.instance_id, + ) + original = instance.root.with_name("original-instance-root") + instance.root.rename(original) + instance.root.mkdir() + with self.assertRaisesRegex( + provisioner_module.system_instance_lifecycle.SystemInstanceLifecycleError, + "replaced", + ): + self.provisioner.purge( + instance.instance_id, "purge-replaced-root", + confirm_instance_id=instance.instance_id, + ) + self.assertTrue(instance.root.exists()) + self.assertTrue(original.exists()) + def test_purge_recovers_after_remove_completed_before_lifecycle_projection(self) -> None: self._create("ep-production-0001", "EP production", 8765) instance = self._instance("ep-production-0001", "EP production") diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py new file mode 100644 index 00000000..c6858b77 --- /dev/null +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +"""Observe a real SIGKILL during exact-wheel EP system-instance purge. + +The only child service adapter writes disposable plist files below the selected +temporary root. The installed wheel supplies the product provisioner and +lifecycle code; this harness never imports the checkout's package. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import plistlib +import signal +import subprocess +import sys +import tempfile +import time +from types import SimpleNamespace + + +ALPHA = "ep-crash-alpha" +BRAVO = "ep-crash-bravo" +OPERATION = "purge-crash-alpha" + + +def _digest_file(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def _tree_digest(root: Path) -> str: + rows: list[tuple[str, str]] = [] + for path in sorted(root.rglob("*")): + relative = path.relative_to(root).as_posix() + if path.is_symlink(): + raise RuntimeError("disposable sibling gained a symbolic link") + rows.append((relative, _digest_file(path) if path.is_file() else "DIRECTORY")) + return "sha256:" + hashlib.sha256(json.dumps(rows).encode()).hexdigest() + + +def _child(mode: str, root: Path, wheel: Path, source: str, files: int) -> None: + import engineering_platform + from engineering_platform import system_instance_provisioner as provisioner + from engineering_platform import system_installation_topology as topology + from engineering_platform import system_provider_context as providers + from engineering_platform import system_server_service as services + + if not Path(engineering_platform.__file__).resolve().is_relative_to(Path(sys.prefix).resolve()): + raise RuntimeError("EP child imported outside its exact-wheel installation") + + launch_root = root / "LaunchDaemons" + launch_root.mkdir(parents=True, exist_ok=True) + + class Controller: + def register(self, instance, interpreter): + paths = services.instance_default_paths(instance, launch_root) + definition = services.instance_service_definition(instance, interpreter=interpreter) + paths.plist_path.write_bytes(plistlib.dumps(services.instance_plist_payload(paths, definition))) + return {"result": "REGISTERED", "label": instance.service_label} + + def quiesce(self, instance): + return {"result": "QUIESCED"} + + def start(self, instance): + return {"result": "RUNNING"} + + def loaded(self, instance): + return False + + def remove(self, instance): + services.instance_default_paths(instance, launch_root).plist_path.unlink(missing_ok=True) + return {"result": "REMOVED"} + + accounts = {ALPHA: "_ep_crash_alpha", BRAVO: "_ep_crash_bravo"} + engine = provisioner.SystemInstanceProvisioner( + root / "product", controller=Controller(), launch_daemons_dir=launch_root, + account_lookup=lambda name: SimpleNamespace(pw_uid=501) if name in accounts.values() + else (_ for _ in ()).throw(KeyError(name)), + health_verifier=lambda instance, interpreter: { + "result": "PASS", "instance_id": instance.instance_id, + "interpreter": str(interpreter), "identity_aware": True, + }, + ) + release = provisioner.ReleaseRequest("2.3.104", wheel, _digest_file(wheel), source) + + if mode == "setup": + for identity, port in ((ALPHA, 18765), (BRAVO, 18766)): + instance = topology.system_instance_topology( + engine.product, instance_id=identity, display_label=identity, + service_account=accounts[identity], + ) + for context in providers.provider_contexts(instance): + context.executable.parent.mkdir(parents=True, exist_ok=True) + context.executable.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + context.executable.chmod(0o755) + engine.provider_register( + instance_id=identity, display_label=identity, + service_account=accounts[identity], provider=context.provider, + executable_sha256=providers.executable_digest(context.executable), + version="qualification-1", + auth_reference=f"{context.provider}:auth:{identity}", + auth_bootstrap_receipt=f"{context.provider}:receipt:{identity}", + ) + engine.create(provisioner.InstanceRequest( + "install-" + identity, identity, identity, accounts[identity], port, release, + )) + payload = root / "product" / "instances" / ALPHA / "crash-sentinel" + payload.mkdir(mode=0o700) + for index in range(files): + (payload / f"file-{index:05d}").write_bytes(b"sentinel\n") + print("SETUP_COMPLETE", flush=True) + elif mode == "purge": + result = engine.purge(ALPHA, OPERATION, confirm_instance_id=ALPHA) + print(json.dumps(result, sort_keys=True), flush=True) + else: + raise RuntimeError("invalid child mode") + + +def _run_child(python: Path, script: Path, mode: str, root: Path, wheel: Path, + source: str, files: int) -> subprocess.CompletedProcess[str]: + return subprocess.run( + (str(python), "-I", str(script), "--internal", mode, str(root), str(wheel), + source, str(files)), + cwd=root, env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "PYTHONNOUSERSITE": "1", + "PYTHONSAFEPATH": "1"}, + text=True, capture_output=True, check=False, + ) + + +def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[str, object]: + wheel = wheel.resolve(strict=True) + if _digest_file(wheel) != expected_digest: + raise RuntimeError("selected EP wheel digest does not match the declared candidate") + if sys.version_info[:2] != (3, 14): + raise RuntimeError("EP process-crash qualification requires Python 3.14") + script = Path(__file__).resolve(strict=True) + with tempfile.TemporaryDirectory(prefix="ep-purge-sigkill-") as temporary: + root = Path(temporary) + venv = root / "bootstrap-venv" + made = subprocess.run((sys.executable, "-I", "-m", "venv", str(venv)), + text=True, capture_output=True, check=False) + if made.returncode: + raise RuntimeError("disposable Python 3.14 venv creation failed") + python = venv / "bin" / "python" + installed = subprocess.run( + (str(python), "-I", "-m", "pip", "install", "--isolated", "--no-deps", + "--no-index", str(wheel)), + text=True, capture_output=True, check=False, + ) + if installed.returncode: + raise RuntimeError("exact EP wheel installation failed") + setup = _run_child(python, script, "setup", root, wheel, source, files) + if setup.returncode or setup.stdout.strip() != "SETUP_COMPLETE": + raise RuntimeError("disposable EP two-instance setup failed: " + setup.stderr[-300:]) + + product = root / "product" + sibling = product / "instances" / BRAVO + sibling_before = _tree_digest(sibling) + sentinel = product / "instances" / ALPHA / "crash-sentinel" + state_path = product / "instance-lifecycle-v1" / ALPHA / "operations" / OPERATION / "state.json" + command = (str(python), "-I", str(script), "--internal", "purge", str(root), + str(wheel), source, str(files)) + child = subprocess.Popen( + command, cwd=root, env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", + "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1"}, + text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, start_new_session=True, + ) + observed = None + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and child.poll() is None: + try: + state = json.loads(state_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + state = {} + if state.get("phase") == "PURGE_READY" and sentinel.is_dir(): + remaining = len(os.listdir(sentinel)) + if 0 < remaining < files and child.poll() is None: + observed = {"phase": "PURGE_READY", "files_before": files, + "files_at_signal": remaining} + os.kill(child.pid, signal.SIGKILL) + break + time.sleep(0.001) + stdout, stderr = child.communicate(timeout=15) + if observed is None or child.returncode != -signal.SIGKILL: + return {"result": "NOT_HIT", "observed": observed, + "exit_kind": "OTHER", "child_error": stderr[-200:]} + process_table = subprocess.run( + ("/bin/ps", "-axo", "pid=,ppid="), text=True, + capture_output=True, check=False, + ) + descendants = 0 + if process_table.returncode == 0: + for line in process_table.stdout.splitlines(): + fields = line.split() + if len(fields) == 2 and fields[1].isdigit() and int(fields[1]) == child.pid: + descendants += 1 + else: + return {"result": "UNPROVEN", "reason": "DESCENDANT_READBACK_UNAVAILABLE"} + if descendants: + return {"result": "UNPROVEN", "reason": "OWNED_DESCENDANT_SURVIVED", + "owned_descendants_alive": descendants} + after_kill = len(os.listdir(sentinel)) if sentinel.is_dir() else 0 + if not 0 < after_kill < files: + return {"result": "UNPROVEN", "observed": observed, + "files_after_kill": after_kill} + if _tree_digest(sibling) != sibling_before: + return {"result": "FAIL", "reason": "SIBLING_CHANGED_AFTER_KILL"} + resumed = _run_child(python, script, "purge", root, wheel, source, files) + tombstone = product / "instance-lifecycle-v1" / ALPHA / "purged.json" + selected = product / "instances" / ALPHA + sibling_after = _tree_digest(sibling) + result: dict[str, object] = { + "result": "PASS" if resumed.returncode == 0 else "GAP_PROVEN", + "wheel_sha256": expected_digest, + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "source_checkout_import": False, + "mutation_phase": observed["phase"], + "signal_exit": "SIGKILL", + "owned_descendants_alive": descendants, + "files_before": files, + "files_at_signal": observed["files_at_signal"], + "files_after_kill": after_kill, + "same_operation_replay": "COMPLETE" if resumed.returncode == 0 else "FAILED", + "selected_root_exists": selected.exists(), + "tombstone_exists": tombstone.exists(), + "sibling_byte_identical": sibling_after == sibling_before, + } + if resumed.returncode == 0: + receipt = json.loads(resumed.stdout) + result["lifecycle_state"] = receipt["receipt"]["evidence"]["lifecycle_state"] + again = _run_child(python, script, "purge", root, wheel, source, files) + result["terminal_replay_identical"] = ( + again.returncode == 0 and json.loads(again.stdout) == receipt + ) + if (result["lifecycle_state"] != "PURGED" or selected.exists() + or not tombstone.exists() or not result["sibling_byte_identical"] + or not result["terminal_replay_identical"]): + result["result"] = "FAIL" + else: + result["resume_error_class"] = resumed.stderr.splitlines()[-1].split(":", 1)[0] + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--wheel", type=Path) + parser.add_argument("--wheel-sha256") + parser.add_argument("--source-revision") + parser.add_argument("--files", type=int, default=20000) + parser.add_argument("--evidence", type=Path) + parser.add_argument("--internal", nargs=5, metavar=("MODE", "ROOT", "WHEEL", "SOURCE", "FILES")) + args = parser.parse_args() + if args.internal is not None: + mode, root, wheel, source, files = args.internal + _child(mode, Path(root), Path(wheel), source, int(files)) + return 0 + if (args.wheel is None or args.wheel_sha256 is None or args.source_revision is None + or args.evidence is None or not 1000 <= args.files <= 50000): + parser.error("exact wheel, digest, source, evidence and bounded file count are required") + result = qualify(args.wheel, args.wheel_sha256, args.source_revision, args.files) + args.evidence.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, sort_keys=True)) + return 0 if result["result"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From da2572bb8aedc99ab6a14cdaa72920e0dc5d2305 Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 08:32:31 +0200 Subject: [PATCH 2/6] Parameterize EP crash harness for released requalification --- .../system_instance_purge_sigkill.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py index c6858b77..4e396185 100644 --- a/tools/qualification/system_instance_purge_sigkill.py +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -41,7 +41,7 @@ def _tree_digest(root: Path) -> str: return "sha256:" + hashlib.sha256(json.dumps(rows).encode()).hexdigest() -def _child(mode: str, root: Path, wheel: Path, source: str, files: int) -> None: +def _child(mode: str, root: Path, wheel: Path, source: str, version: str, files: int) -> None: import engineering_platform from engineering_platform import system_instance_provisioner as provisioner from engineering_platform import system_installation_topology as topology @@ -84,7 +84,7 @@ def remove(self, instance): "interpreter": str(interpreter), "identity_aware": True, }, ) - release = provisioner.ReleaseRequest("2.3.104", wheel, _digest_file(wheel), source) + release = provisioner.ReleaseRequest(version, wheel, _digest_file(wheel), source) if mode == "setup": for identity, port in ((ALPHA, 18765), (BRAVO, 18766)): @@ -120,17 +120,18 @@ def remove(self, instance): def _run_child(python: Path, script: Path, mode: str, root: Path, wheel: Path, - source: str, files: int) -> subprocess.CompletedProcess[str]: + source: str, version: str, files: int) -> subprocess.CompletedProcess[str]: return subprocess.run( (str(python), "-I", str(script), "--internal", mode, str(root), str(wheel), - source, str(files)), + source, version, str(files)), cwd=root, env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1"}, text=True, capture_output=True, check=False, ) -def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[str, object]: +def qualify(wheel: Path, expected_digest: str, source: str, version: str, + files: int) -> dict[str, object]: wheel = wheel.resolve(strict=True) if _digest_file(wheel) != expected_digest: raise RuntimeError("selected EP wheel digest does not match the declared candidate") @@ -152,7 +153,7 @@ def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[ ) if installed.returncode: raise RuntimeError("exact EP wheel installation failed") - setup = _run_child(python, script, "setup", root, wheel, source, files) + setup = _run_child(python, script, "setup", root, wheel, source, version, files) if setup.returncode or setup.stdout.strip() != "SETUP_COMPLETE": raise RuntimeError("disposable EP two-instance setup failed: " + setup.stderr[-300:]) @@ -162,7 +163,7 @@ def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[ sentinel = product / "instances" / ALPHA / "crash-sentinel" state_path = product / "instance-lifecycle-v1" / ALPHA / "operations" / OPERATION / "state.json" command = (str(python), "-I", str(script), "--internal", "purge", str(root), - str(wheel), source, str(files)) + str(wheel), source, version, str(files)) child = subprocess.Popen( command, cwd=root, env={"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1"}, @@ -208,13 +209,14 @@ def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[ "files_after_kill": after_kill} if _tree_digest(sibling) != sibling_before: return {"result": "FAIL", "reason": "SIBLING_CHANGED_AFTER_KILL"} - resumed = _run_child(python, script, "purge", root, wheel, source, files) + resumed = _run_child(python, script, "purge", root, wheel, source, version, files) tombstone = product / "instance-lifecycle-v1" / ALPHA / "purged.json" selected = product / "instances" / ALPHA sibling_after = _tree_digest(sibling) result: dict[str, object] = { "result": "PASS" if resumed.returncode == 0 else "GAP_PROVEN", "wheel_sha256": expected_digest, + "version": version, "python": f"{sys.version_info.major}.{sys.version_info.minor}", "source_checkout_import": False, "mutation_phase": observed["phase"], @@ -231,7 +233,7 @@ def qualify(wheel: Path, expected_digest: str, source: str, files: int) -> dict[ if resumed.returncode == 0: receipt = json.loads(resumed.stdout) result["lifecycle_state"] = receipt["receipt"]["evidence"]["lifecycle_state"] - again = _run_child(python, script, "purge", root, wheel, source, files) + again = _run_child(python, script, "purge", root, wheel, source, version, files) result["terminal_replay_identical"] = ( again.returncode == 0 and json.loads(again.stdout) == receipt ) @@ -249,18 +251,20 @@ def main() -> int: parser.add_argument("--wheel", type=Path) parser.add_argument("--wheel-sha256") parser.add_argument("--source-revision") + parser.add_argument("--version") parser.add_argument("--files", type=int, default=20000) parser.add_argument("--evidence", type=Path) - parser.add_argument("--internal", nargs=5, metavar=("MODE", "ROOT", "WHEEL", "SOURCE", "FILES")) + parser.add_argument("--internal", nargs=6, metavar=("MODE", "ROOT", "WHEEL", "SOURCE", "VERSION", "FILES")) args = parser.parse_args() if args.internal is not None: - mode, root, wheel, source, files = args.internal - _child(mode, Path(root), Path(wheel), source, int(files)) + mode, root, wheel, source, version, files = args.internal + _child(mode, Path(root), Path(wheel), source, version, int(files)) return 0 if (args.wheel is None or args.wheel_sha256 is None or args.source_revision is None + or args.version is None or args.evidence is None or not 1000 <= args.files <= 50000): parser.error("exact wheel, digest, source, evidence and bounded file count are required") - result = qualify(args.wheel, args.wheel_sha256, args.source_revision, args.files) + result = qualify(args.wheel, args.wheel_sha256, args.source_revision, args.version, args.files) args.evidence.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") print(json.dumps(result, sort_keys=True)) return 0 if result["result"] == "PASS" else 1 From e1c0b4e0d5172c11f6febf6020ffa7c1953e8230 Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 08:53:33 +0200 Subject: [PATCH 3/6] Qualify EP lifecycle crash phase matrix on installed wheels --- .../EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md | 20 +- .../system_instance_lifecycle_sigkill.py | 232 ++++++++++++++++++ .../system_instance_purge_sigkill.py | 9 + 3 files changed, 258 insertions(+), 3 deletions(-) create mode 100644 tools/qualification/system_instance_lifecycle_sigkill.py diff --git a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md index acd53b77..c47ad4d4 100644 --- a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md +++ b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md @@ -315,9 +315,23 @@ is `tools/qualification/system_instance_purge_sigkill.py`; it reports durable `PURGE_READY`, physical partial deletion, exact mutator signal exit, surviving descendants, same-operation recovery, terminal replay and sibling byte identity. +`tools/qualification/system_instance_lifecycle_sigkill.py` exercises the +product-owned process-stop matrix through the installed provisioner using two +disposable instances and a filesystem-only service adapter. Its 2.3.104 +published-wheel runs observed `PREPARED`, `VERIFIED`, receipt-before-`COMPLETE` +and `COMPLETE` for PRESERVE and RESTORE. They observed `PREPARED`, `PURGE_READY`, +terminal REMOVE receipt, tombstone-before-receipt, lifecycle receipt-before- +`COMPLETE` and `COMPLETE` for PURGE. Each of these cells resumed the same +operation to the expected terminal status with identical replay and unchanged +sibling bytes. The separate partial physical deletion cell remains +`GAP_PROVEN` on published 2.3.104 and passed only with the local correction +candidate. A missed short phase is recorded as `NOT_HIT`, never promoted to a +hit based on timing. + Source tests and a locally built candidate-wheel SIGKILL run have passed this specific partial-deletion cell. The new protected source, any required normal release and requalification against its published bytes are still required -before this process-crash subset may be called qualified. Other revision-26 -PRESERVE/RESTORE/PURGE crash cells remain separately open; no physical reboot, -power-loss, production service or installer acceptance is inferred. +before this process-crash subset may be called qualified. The remaining +conflict, foreign-request and published-fixed-wheel checks must complete +before the entire revision-26 subset can be called qualified. No physical +reboot, power-loss, production service or installer acceptance is inferred. diff --git a/tools/qualification/system_instance_lifecycle_sigkill.py b/tools/qualification/system_instance_lifecycle_sigkill.py new file mode 100644 index 00000000..4541bfe6 --- /dev/null +++ b/tools/qualification/system_instance_lifecycle_sigkill.py @@ -0,0 +1,232 @@ +#!/usr/bin/env python3 +"""Exact-wheel EP preserve/restore/purge SIGKILL phase qualification. + +The supervised child enters the installed product through the normal +SystemInstanceProvisioner API. The local service adapter writes only inside +its disposable root. Unobserved boundaries remain NOT_HIT. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import tempfile +import time + + +ALPHA = "ep-crash-alpha" +BRAVO = "ep-crash-bravo" +OPERATIONS = {"preserve": "preserve-crash-alpha", "restore": "restore-crash-alpha", "purge": "purge-crash-alpha"} +BOUNDARIES = { + "preserve": ("PREPARED", "VERIFIED", "RECEIPT", "COMPLETE"), + "restore": ("PREPARED", "VERIFIED", "RECEIPT", "COMPLETE"), + "purge": ("PREPARED", "PURGE_READY", "REMOVE_RECEIPT", "PARTIAL_DELETE", "TOMBSTONE", "RECEIPT", "COMPLETE"), +} +ENV = {"PATH": "/usr/bin:/bin:/usr/sbin:/sbin", "PYTHONNOUSERSITE": "1", "PYTHONSAFEPATH": "1"} + + +def file_digest(path: Path) -> str: + return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_digest(root: Path) -> str: + rows = [] + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise RuntimeError("disposable sibling gained a symlink") + rows.append((path.relative_to(root).as_posix(), file_digest(path) if path.is_file() else "DIRECTORY")) + return "sha256:" + hashlib.sha256(json.dumps(rows).encode()).hexdigest() + + +def auth_states(instance_root: Path) -> dict[str, object]: + result = {} + for provider in ("codex", "github"): + path = instance_root / "providers" / provider / "auth-state.json" + result[provider] = json.loads(path.read_text())["state"] if path.exists() else "ABSENT" + return result + + +def child_command(python: Path, helper: Path, mode: str, root: Path, wheel: Path, + source: str, version: str, files: int) -> list[str]: + return [str(python), "-I", str(helper), "--internal", mode, str(root), + str(wheel), source, version, str(files)] + + +def run_child(command: list[str], root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=root, env=ENV, capture_output=True, text=True, check=False) + + +def require_child(command: list[str], root: Path, label: str) -> None: + result = run_child(command, root) + if result.returncode: + raise RuntimeError(label + " failed: " + result.stderr[-300:]) + + +def read_state(path: Path) -> dict[str, object]: + try: + value = json.loads(path.read_text()) + except (OSError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + +def observed_boundary(boundary: str, phase: object, receipt: Path, remove_receipt: Path, + tombstone: Path, sentinel: Path, files: int) -> dict[str, object] | None: + if boundary in ("PREPARED", "VERIFIED", "PURGE_READY", "COMPLETE"): + return {"phase": phase} if phase == boundary else None + if boundary == "RECEIPT": + return {"phase": phase, "receipt_observed": True} if receipt.exists() and phase != "COMPLETE" else None + if boundary == "REMOVE_RECEIPT": + return {"phase": phase, "remove_receipt_observed": True} if phase == "PURGE_READY" and remove_receipt.exists() else None + if boundary == "TOMBSTONE": + return {"phase": phase, "tombstone_observed": True} if tombstone.exists() and not receipt.exists() else None + if boundary == "PARTIAL_DELETE" and phase == "PURGE_READY" and sentinel.is_dir(): + remaining = len(os.listdir(sentinel)) + if 0 < remaining < files: + return {"phase": phase, "files_at_signal": remaining, "files_before": files} + return None + + +def qualify(wheel: Path, expected_digest: str, version: str, source: str, + operation: str, boundary: str, files: int) -> dict[str, object]: + if sys.version_info[:2] != (3, 14): + raise RuntimeError("EP qualification requires Python 3.14 without bypass") + if operation not in BOUNDARIES or boundary not in BOUNDARIES[operation]: + raise RuntimeError("operation or durable boundary is not registered") + wheel = wheel.resolve(strict=True) + if file_digest(wheel) != expected_digest: + raise RuntimeError("exact EP wheel digest mismatch") + helper = Path(__file__).with_name("system_instance_purge_sigkill.py").resolve(strict=True) + with tempfile.TemporaryDirectory(prefix="ep-lifecycle-sigkill-") as temporary: + root = Path(temporary) + venv = root / "bootstrap-venv" + made = subprocess.run((sys.executable, "-I", "-m", "venv", str(venv)), + capture_output=True, text=True, check=False) + if made.returncode: + raise RuntimeError("disposable Python 3.14 environment failed") + python = venv / "bin" / "python" + installed = subprocess.run((str(python), "-I", "-m", "pip", "install", "--isolated", + "--no-deps", "--no-index", str(wheel)), + capture_output=True, text=True, check=False) + if installed.returncode: + raise RuntimeError("exact EP wheel installation failed") + command = lambda mode: child_command(python, helper, mode, root, wheel, source, version, files) + require_child(command("setup"), root, "two-instance setup") + if operation == "restore": + require_child(command("preserve"), root, "normal preserve prerequisite") + + product = root / "product" + selected = product / "instances" / ALPHA + sibling = product / "instances" / BRAVO + sibling_before = tree_digest(sibling) + op_id = OPERATIONS[operation] + operation_root = product / "instance-lifecycle-v1" / ALPHA / "operations" / op_id + state_path = operation_root / "state.json" + receipt_path = operation_root / "receipt.json" + remove_receipt = product / "receipts" / ALPHA / (op_id + ".json") + tombstone = product / "instance-lifecycle-v1" / ALPHA / "purged.json" + sentinel = selected / "crash-sentinel" + detached_sentinel = selected.parent / "missing" + if operation == "purge": + # Legacy REMOVE first renames or removes the instance tree directly; + # observe deletion in its original root using the product's state. + detached_sentinel = sentinel + child = subprocess.Popen(command(operation), cwd=root, env=ENV, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, start_new_session=True) + observed = None + deadline = time.monotonic() + 60 + while time.monotonic() < deadline and child.poll() is None: + state = read_state(state_path) + observed = observed_boundary(boundary, state.get("phase"), receipt_path, + remove_receipt, tombstone, detached_sentinel, files) + if observed is not None and child.poll() is None: + os.kill(child.pid, signal.SIGKILL) + break + observed = None + time.sleep(0) + _, stderr = child.communicate(timeout=30) + after = read_state(state_path) + process_table = subprocess.run(("/bin/ps", "-axo", "pid=,ppid="), + capture_output=True, text=True, check=False) + if process_table.returncode: + return {"result": "UNPROVEN", "reason": "descendant readback unavailable"} + descendants = sum(1 for line in process_table.stdout.splitlines() + if len(line.split()) == 2 and line.split()[1] == str(child.pid)) + result: dict[str, object] = { + "operation": operation, "boundary": boundary, "version": version, + "wheel_sha256": expected_digest, "source_checkout_import": False, + "observed": observed, "phase_after_kill": after.get("phase"), + "signal_exit": "SIGKILL" if child.returncode == -signal.SIGKILL else "OTHER", + "owned_descendants_alive": descendants, + "remove_receipt_after_kill": remove_receipt.exists(), + "lifecycle_receipt_after_kill": receipt_path.exists(), + "tombstone_after_kill": tombstone.exists(), + "selected_root_after_kill": selected.exists(), + "service_definition_count_after_kill": len(list((root / "LaunchDaemons").glob("*.plist"))), + "provider_states_after_kill": auth_states(selected) if selected.exists() else {"codex": "ABSENT", "github": "ABSENT"}, + "sibling_byte_identical_after_kill": tree_digest(sibling) == sibling_before, + } + if boundary == "PARTIAL_DELETE": + result["files_after_kill"] = len(os.listdir(sentinel)) if sentinel.is_dir() else 0 + if (observed is None or child.returncode != -signal.SIGKILL or descendants + or after.get("phase") != observed["phase"]): + result.update(result="NOT_HIT", child_error=stderr[-200:]) + return result + if boundary == "PARTIAL_DELETE" and not 0 < result["files_after_kill"] < files: + result.update(result="UNPROVEN", reason="partial physical deletion not retained") + return result + resumed = run_child(command(operation), root) + if resumed.returncode: + result.update(result="GAP_PROVEN", resume_error_class=resumed.stderr.splitlines()[-1].split(":", 1)[0]) + return result + terminal = json.loads(resumed.stdout) + receipt_bytes = receipt_path.read_bytes() + replay = run_child(command(operation), root) + identical = (replay.returncode == 0 and json.loads(replay.stdout) == terminal + and receipt_path.read_bytes() == receipt_bytes) + evidence = terminal["receipt"]["evidence"] + provider_states = auth_states(selected) if selected.exists() else {"codex": "ABSENT", "github": "ABSENT"} + expect_purged = operation == "purge" + okay = (identical and tree_digest(sibling) == sibling_before + and selected.exists() != expect_purged and tombstone.exists() == expect_purged + and ((evidence.get("lifecycle_state") == "PURGED" and evidence.get("restorable") is False) + if expect_purged else all(value == "PRESERVED_REQUIRES_REVERIFICATION" for value in provider_states.values())) + and (operation != "restore" or evidence.get("service_state") == "REGISTERED_INACTIVE" + and evidence.get("ready") is False)) + result.update(result="PASS" if okay else "FAIL", terminal_replay_identical=identical, + sibling_byte_identical=tree_digest(sibling) == sibling_before, + selected_root_final_exists=selected.exists(), tombstone_final_exists=tombstone.exists(), + provider_states_final=provider_states, + lifecycle_state=evidence.get("lifecycle_state"), + service_state=evidence.get("service_state"), ready=evidence.get("ready")) + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--wheel", type=Path, required=True) + parser.add_argument("--wheel-sha256", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-revision", required=True) + parser.add_argument("--operation", choices=tuple(OPERATIONS), required=True) + parser.add_argument("--boundary", required=True) + parser.add_argument("--files", type=int, default=1000) + parser.add_argument("--evidence", type=Path, required=True) + args = parser.parse_args() + if not 1000 <= args.files <= 50000: + parser.error("bounded file count must be between 1000 and 50000") + result = qualify(args.wheel, args.wheel_sha256, args.version, args.source_revision, + args.operation, args.boundary, args.files) + args.evidence.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps(result, sort_keys=True)) + return 0 if result["result"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py index 4e396185..1fe9482c 100644 --- a/tools/qualification/system_instance_purge_sigkill.py +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -115,6 +115,15 @@ def remove(self, instance): elif mode == "purge": result = engine.purge(ALPHA, OPERATION, confirm_instance_id=ALPHA) print(json.dumps(result, sort_keys=True), flush=True) + elif mode == "preserve": + result = engine.preserve(ALPHA, "preserve-crash-alpha", confirm_instance_id=ALPHA) + print(json.dumps(result, sort_keys=True), flush=True) + elif mode == "restore": + result = engine.restore( + ALPHA, "restore-crash-alpha", preserve_operation_id="preserve-crash-alpha", + release=release, + ) + print(json.dumps(result, sort_keys=True), flush=True) else: raise RuntimeError("invalid child mode") From 31f27bd0170e40bf141312f9daca70534e5adf7a Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 09:08:43 +0200 Subject: [PATCH 4/6] Check EP lifecycle conflict and foreign recovery controls --- .../EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md | 5 +++- .../system_instance_lifecycle_sigkill.py | 26 ++++++++++++++++++- .../system_instance_purge_sigkill.py | 25 ++++++++++++++++++ 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md index c47ad4d4..30cdd32b 100644 --- a/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md +++ b/docs/engineering/EP_SERVER_SYSTEM_MULTI_INSTANCE_V1.md @@ -326,7 +326,10 @@ operation to the expected terminal status with identical replay and unchanged sibling bytes. The separate partial physical deletion cell remains `GAP_PROVEN` on published 2.3.104 and passed only with the local correction candidate. A missed short phase is recorded as `NOT_HIT`, never promoted to a -hit based on timing. +hit based on timing. The reusable harness also checks a held live lifecycle +lock, changed request, foreign operation/instance status, OS-released lock +reacquisition through normal retry, provider demotion, inactive restore, +terminal replay and restore rejection after purge. Source tests and a locally built candidate-wheel SIGKILL run have passed this specific partial-deletion cell. The new protected source, any required normal diff --git a/tools/qualification/system_instance_lifecycle_sigkill.py b/tools/qualification/system_instance_lifecycle_sigkill.py index 4541bfe6..5d02fe99 100644 --- a/tools/qualification/system_instance_lifecycle_sigkill.py +++ b/tools/qualification/system_instance_lifecycle_sigkill.py @@ -8,6 +8,7 @@ from __future__ import annotations import argparse +import fcntl import hashlib import json import os @@ -123,6 +124,14 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, product = root / "product" selected = product / "instances" / ALPHA sibling = product / "instances" / BRAVO + lifecycle_lock = selected / "locks" / "lifecycle.lock" + lifecycle_lock.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + with lifecycle_lock.open("a+b") as lock_stream: + fcntl.flock(lock_stream, fcntl.LOCK_EX | fcntl.LOCK_NB) + live_conflict_rejected = run_child(command("conflict-" + operation), root).returncode != 0 + fcntl.flock(lock_stream, fcntl.LOCK_UN) + if not live_conflict_rejected: + raise RuntimeError("normal provisioner accepted a competing live lock owner") sibling_before = tree_digest(sibling) op_id = OPERATIONS[operation] operation_root = product / "instance-lifecycle-v1" / ALPHA / "operations" / op_id @@ -164,6 +173,7 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, "observed": observed, "phase_after_kill": after.get("phase"), "signal_exit": "SIGKILL" if child.returncode == -signal.SIGKILL else "OTHER", "owned_descendants_alive": descendants, + "live_conflict_rejected": live_conflict_rejected, "remove_receipt_after_kill": remove_receipt.exists(), "lifecycle_receipt_after_kill": receipt_path.exists(), "tombstone_after_kill": tombstone.exists(), @@ -181,6 +191,9 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, if boundary == "PARTIAL_DELETE" and not 0 < result["files_after_kill"] < files: result.update(result="UNPROVEN", reason="partial physical deletion not retained") return result + changed_request_rejected = run_child(command("changed-" + operation), root).returncode != 0 + foreign_operation_rejected = run_child(command("foreign-operation-" + operation), root).returncode != 0 + foreign_instance_rejected = run_child(command("foreign-instance-" + operation), root).returncode != 0 resumed = run_child(command(operation), root) if resumed.returncode: result.update(result="GAP_PROVEN", resume_error_class=resumed.stderr.splitlines()[-1].split(":", 1)[0]) @@ -193,13 +206,24 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, evidence = terminal["receipt"]["evidence"] provider_states = auth_states(selected) if selected.exists() else {"codex": "ABSENT", "github": "ABSENT"} expect_purged = operation == "purge" - okay = (identical and tree_digest(sibling) == sibling_before + restore_after_purge_rejected = ( + run_child(command("restore-after-purge"), root).returncode != 0 + if expect_purged else None + ) + okay = (identical and changed_request_rejected and foreign_operation_rejected + and foreign_instance_rejected and live_conflict_rejected + and (not expect_purged or restore_after_purge_rejected) + and tree_digest(sibling) == sibling_before and selected.exists() != expect_purged and tombstone.exists() == expect_purged and ((evidence.get("lifecycle_state") == "PURGED" and evidence.get("restorable") is False) if expect_purged else all(value == "PRESERVED_REQUIRES_REVERIFICATION" for value in provider_states.values())) and (operation != "restore" or evidence.get("service_state") == "REGISTERED_INACTIVE" and evidence.get("ready") is False)) result.update(result="PASS" if okay else "FAIL", terminal_replay_identical=identical, + changed_request_rejected=changed_request_rejected, + foreign_operation_status_rejected=foreign_operation_rejected, + foreign_instance_status_rejected=foreign_instance_rejected, + restore_after_purge_rejected=restore_after_purge_rejected, sibling_byte_identical=tree_digest(sibling) == sibling_before, selected_root_final_exists=selected.exists(), tombstone_final_exists=tombstone.exists(), provider_states_final=provider_states, diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py index 1fe9482c..20c3be85 100644 --- a/tools/qualification/system_instance_purge_sigkill.py +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -124,6 +124,31 @@ def remove(self, instance): release=release, ) print(json.dumps(result, sort_keys=True), flush=True) + elif mode == "conflict-preserve": + engine.preserve(ALPHA, "conflict-preflight", confirm_instance_id=ALPHA) + elif mode == "conflict-restore": + engine.restore(ALPHA, "conflict-preflight", preserve_operation_id="preserve-crash-alpha", + release=release) + elif mode == "conflict-purge": + engine.purge(ALPHA, "conflict-preflight", confirm_instance_id=ALPHA) + elif mode == "changed-preserve": + engine.preserve(ALPHA, "preserve-crash-alpha", confirm_instance_id=BRAVO) + elif mode == "changed-restore": + changed = provisioner.ReleaseRequest(version, wheel, _digest_file(wheel), "0" * 40) + engine.restore(ALPHA, "restore-crash-alpha", preserve_operation_id="preserve-crash-alpha", + release=changed) + elif mode == "changed-purge": + engine.purge(ALPHA, OPERATION, confirm_instance_id=BRAVO) + elif mode.startswith("foreign-operation-"): + engine.lifecycle_status(ALPHA, "foreign-operation-no-evidence") + elif mode.startswith("foreign-instance-"): + operation = mode.removeprefix("foreign-instance-") + operation_id = {"preserve": "preserve-crash-alpha", "restore": "restore-crash-alpha", + "purge": OPERATION}[operation] + engine.lifecycle_status(BRAVO, operation_id) + elif mode == "restore-after-purge": + engine.restore(ALPHA, "restore-after-purge", preserve_operation_id="preserve-crash-alpha", + release=release) else: raise RuntimeError("invalid child mode") From a52912db1c138cdb12a0c1d50111adfa5f2def3d Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 09:14:00 +0200 Subject: [PATCH 5/6] Terminate only owned EP test process groups --- tools/qualification/system_instance_lifecycle_sigkill.py | 9 +++++++-- tools/qualification/system_instance_purge_sigkill.py | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tools/qualification/system_instance_lifecycle_sigkill.py b/tools/qualification/system_instance_lifecycle_sigkill.py index 5d02fe99..d89fc1ca 100644 --- a/tools/qualification/system_instance_lifecycle_sigkill.py +++ b/tools/qualification/system_instance_lifecycle_sigkill.py @@ -155,13 +155,18 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, observed = observed_boundary(boundary, state.get("phase"), receipt_path, remove_receipt, tombstone, detached_sentinel, files) if observed is not None and child.poll() is None: - os.kill(child.pid, signal.SIGKILL) + try: + if os.getpgid(child.pid) != child.pid: + raise RuntimeError("EP child is not its owned process-group leader") + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + observed = None break observed = None time.sleep(0) _, stderr = child.communicate(timeout=30) after = read_state(state_path) - process_table = subprocess.run(("/bin/ps", "-axo", "pid=,ppid="), + process_table = subprocess.run(("/bin/ps", "-axo", "pid=,pgid="), capture_output=True, text=True, check=False) if process_table.returncode: return {"result": "UNPROVEN", "reason": "descendant readback unavailable"} diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py index 20c3be85..4e6f2c90 100644 --- a/tools/qualification/system_instance_purge_sigkill.py +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -215,7 +215,12 @@ def qualify(wheel: Path, expected_digest: str, source: str, version: str, if 0 < remaining < files and child.poll() is None: observed = {"phase": "PURGE_READY", "files_before": files, "files_at_signal": remaining} - os.kill(child.pid, signal.SIGKILL) + try: + if os.getpgid(child.pid) != child.pid: + raise RuntimeError("EP child is not its owned process-group leader") + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + observed = None break time.sleep(0.001) stdout, stderr = child.communicate(timeout=15) @@ -223,7 +228,7 @@ def qualify(wheel: Path, expected_digest: str, source: str, version: str, return {"result": "NOT_HIT", "observed": observed, "exit_kind": "OTHER", "child_error": stderr[-200:]} process_table = subprocess.run( - ("/bin/ps", "-axo", "pid=,ppid="), text=True, + ("/bin/ps", "-axo", "pid=,pgid="), text=True, capture_output=True, check=False, ) descendants = 0 From db55915a4a42b16dcbbfc2d4f7e74ba60ea59208 Mon Sep 17 00:00:00 2001 From: Peter van Tol Date: Tue, 29 Sep 2026 10:25:47 +0200 Subject: [PATCH 6/6] Wait for owned crash-test descendants to exit --- .../system_instance_lifecycle_sigkill.py | 18 +++++++++----- .../system_instance_purge_sigkill.py | 24 ++++++++++--------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/tools/qualification/system_instance_lifecycle_sigkill.py b/tools/qualification/system_instance_lifecycle_sigkill.py index d89fc1ca..69d0e7e3 100644 --- a/tools/qualification/system_instance_lifecycle_sigkill.py +++ b/tools/qualification/system_instance_lifecycle_sigkill.py @@ -166,12 +166,18 @@ def qualify(wheel: Path, expected_digest: str, version: str, source: str, time.sleep(0) _, stderr = child.communicate(timeout=30) after = read_state(state_path) - process_table = subprocess.run(("/bin/ps", "-axo", "pid=,pgid="), - capture_output=True, text=True, check=False) - if process_table.returncode: - return {"result": "UNPROVEN", "reason": "descendant readback unavailable"} - descendants = sum(1 for line in process_table.stdout.splitlines() - if len(line.split()) == 2 and line.split()[1] == str(child.pid)) + descendants = 0 + for _ in range(100): + process_table = subprocess.run(("/bin/ps", "-axo", "pid=,pgid=,state="), + capture_output=True, text=True, check=False) + if process_table.returncode: + return {"result": "UNPROVEN", "reason": "descendant readback unavailable"} + descendants = sum(1 for line in process_table.stdout.splitlines() + if len(line.split()) == 3 and line.split()[1] == str(child.pid) + and not line.split()[2].startswith("Z")) + if not descendants: + break + time.sleep(0.05) result: dict[str, object] = { "operation": operation, "boundary": boundary, "version": version, "wheel_sha256": expected_digest, "source_checkout_import": False, diff --git a/tools/qualification/system_instance_purge_sigkill.py b/tools/qualification/system_instance_purge_sigkill.py index 4e6f2c90..68f71fa2 100644 --- a/tools/qualification/system_instance_purge_sigkill.py +++ b/tools/qualification/system_instance_purge_sigkill.py @@ -227,18 +227,20 @@ def qualify(wheel: Path, expected_digest: str, source: str, version: str, if observed is None or child.returncode != -signal.SIGKILL: return {"result": "NOT_HIT", "observed": observed, "exit_kind": "OTHER", "child_error": stderr[-200:]} - process_table = subprocess.run( - ("/bin/ps", "-axo", "pid=,pgid="), text=True, - capture_output=True, check=False, - ) descendants = 0 - if process_table.returncode == 0: - for line in process_table.stdout.splitlines(): - fields = line.split() - if len(fields) == 2 and fields[1].isdigit() and int(fields[1]) == child.pid: - descendants += 1 - else: - return {"result": "UNPROVEN", "reason": "DESCENDANT_READBACK_UNAVAILABLE"} + for _ in range(100): + process_table = subprocess.run( + ("/bin/ps", "-axo", "pid=,pgid=,state="), text=True, + capture_output=True, check=False, + ) + if process_table.returncode: + return {"result": "UNPROVEN", "reason": "DESCENDANT_READBACK_UNAVAILABLE"} + descendants = sum(1 for line in process_table.stdout.splitlines() + if len(line.split()) == 3 and line.split()[1] == str(child.pid) + and not line.split()[2].startswith("Z")) + if not descendants: + break + time.sleep(0.05) if descendants: return {"result": "UNPROVEN", "reason": "OWNED_DESCENDANT_SURVIVED", "owned_descendants_alive": descendants}