From 90b25516118f531d7e2d65ab83c44285af289c9c Mon Sep 17 00:00:00 2001 From: Invoker Date: Sat, 12 Sep 2026 19:26:10 +0000 Subject: [PATCH 1/7] Add dormant hook runner metrics --- engine/hooks/_runner/outcome.py | 49 ++++++ engine/hooks/_runner/run.py | 147 ++++++++++++++++++ engine/hooks/_runner/tests/test_outcome.py | 56 +++++++ engine/hooks/_runner/tests/test_run.py | 169 +++++++++++++++++++++ 4 files changed, 421 insertions(+) create mode 100644 engine/hooks/_runner/outcome.py create mode 100644 engine/hooks/_runner/run.py create mode 100644 engine/hooks/_runner/tests/test_outcome.py create mode 100644 engine/hooks/_runner/tests/test_run.py diff --git a/engine/hooks/_runner/outcome.py b/engine/hooks/_runner/outcome.py new file mode 100644 index 00000000..f4e7fed8 --- /dev/null +++ b/engine/hooks/_runner/outcome.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import json + +OUTCOMES = { + "timed_out", + "crashed", + "blocked", + "caught_error", + "spoke", + "silent", +} + + +def _stderr_has_hook_error(stderr: bytes) -> bool: + return any(line.startswith(b"catstack-hook-error ") for line in stderr.splitlines()) + + +def _stdout_blocks(stdout: bytes) -> bool: + try: + payload = json.loads(stdout.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return False + if not isinstance(payload, dict): + return False + if payload.get("decision") == "block": + return True + if payload.get("continue") is False: + return True + hook_output = payload.get("hookSpecificOutput") + if isinstance(hook_output, dict) and hook_output.get("permissionDecision") == "deny": + return True + return payload.get("permission") == "deny" + + +def classify(exit_code: int | None, stdout: bytes, stderr: bytes, timed_out: bool) -> str: + if timed_out: + return "timed_out" + if exit_code == 2: + return "blocked" + if exit_code not in (0, None): + return "crashed" + if _stderr_has_hook_error(stderr): + return "caught_error" + if _stdout_blocks(stdout): + return "blocked" + if stdout.strip(): + return "spoke" + return "silent" diff --git a/engine/hooks/_runner/run.py b/engine/hooks/_runner/run.py new file mode 100644 index 00000000..1cdf2776 --- /dev/null +++ b/engine/hooks/_runner/run.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import argparse +import datetime +import json +import os +import subprocess +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from outcome import classify + + +def _hooks_root() -> str: + return os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def _harness(hooks_root: str) -> str: + for name in ("claude", "cursor", "codex"): + if f"/.{name}/" in hooks_root: + return name + return "unknown" + + +def _stdin_fields(stdin: bytes) -> tuple[str | None, str | None]: + try: + payload = json.loads(stdin.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None, None + if not isinstance(payload, dict): + return None, None + session_id = payload.get("session_id") + if session_id is None: + session_id = payload.get("conversation_id") + return payload.get("hook_event_name"), session_id + + +def _metrics_path() -> str: + root = os.environ.get("CATSTACK_HOOK_METRICS_DIR") + if not root: + root = os.path.expanduser(os.path.join("~", ".cache", "catstack-hook-metrics")) + return os.path.join(root, "runs.jsonl") + + +def _write_metrics(row: dict[str, object], path: str) -> bytes: + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "a", encoding="utf-8") as handle: + handle.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n") + except OSError as exc: + return f"catstack-hook-metrics: could not write row to {path}: {exc}\n".encode() + return b"" + + +def _format_timeout(seconds: float) -> str: + if seconds == int(seconds): + return str(int(seconds)) + return str(seconds) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--timeout", type=float) + parser.add_argument("hook_script") + parser.add_argument("args", nargs=argparse.REMAINDER) + return parser.parse_args(argv) + + +def _row( + hooks_root: str, + hook: str, + script: str, + stdin: bytes, + outcome: str, + exit_code: int | None, + started: float, + stdout: bytes, + stderr: bytes, +) -> dict[str, object]: + event, session_id = _stdin_fields(stdin) + return { + "ts": datetime.datetime.now(datetime.UTC).isoformat(), + "harness": _harness(hooks_root), + "hook": hook, + "script": script, + "event": event, + "session_id": session_id, + "outcome": outcome, + "exit_code": exit_code, + "duration_ms": int((time.monotonic() - started) * 1000), + "stdout_bytes": len(stdout), + "stderr_tail": stderr.decode("utf-8", errors="replace")[-500:], + } + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + started = time.monotonic() + stdin = sys.stdin.buffer.read() + hooks_root = _hooks_root() + hook, script = args.hook_script.split("/", 1) if "/" in args.hook_script else (args.hook_script, "") + script_path = os.path.join(hooks_root, hook, script) + stdout = b"" + stderr = b"" + exit_code = 1 + timed_out = False + + if not script or not os.path.isfile(script_path): + stderr = f"catstack-hook-runner: no such hook script: {script_path}\n".encode() + else: + proc = subprocess.Popen( + [sys.executable, script_path, *args.args], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=os.getcwd(), + env=os.environ.copy(), + ) + try: + stdout, stderr = proc.communicate(stdin, timeout=args.timeout) + exit_code = proc.returncode + except subprocess.TimeoutExpired: + proc.kill() + proc.communicate() + timed_out = True + stdout = b"" + stderr = ( + f"catstack-hook-runner: {args.hook_script} timed out after " + f"{_format_timeout(args.timeout)}s\n" + ).encode() + exit_code = 1 + + outcome = classify(exit_code, stdout, stderr, timed_out) + row = _row(hooks_root, hook, script, stdin, outcome, exit_code, started, stdout, stderr) + metrics_error = _write_metrics(row, _metrics_path()) + sys.stdout.buffer.write(stdout) + sys.stdout.buffer.flush() + sys.stderr.buffer.write(stderr) + sys.stderr.buffer.write(metrics_error) + sys.stderr.buffer.flush() + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/engine/hooks/_runner/tests/test_outcome.py b/engine/hooks/_runner/tests/test_outcome.py new file mode 100644 index 00000000..2eb68d97 --- /dev/null +++ b/engine/hooks/_runner/tests/test_outcome.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import os +import sys +import unittest + +RUNNER_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, RUNNER_DIR) + +import outcome + + +class ClassifyOutcomes(unittest.TestCase): + def test_timed_out_wins(self): + self.assertEqual(outcome.classify(2, b'{"decision":"block"}', b"", True), "timed_out") + + def test_exit_two_blocks_before_crash(self): + self.assertEqual(outcome.classify(2, b"", b"", False), "blocked") + + def test_other_nonzero_exit_crashes(self): + self.assertEqual(outcome.classify(1, b'{"decision":"block"}', b"", False), "crashed") + + def test_stderr_hook_error_is_caught_error(self): + self.assertEqual(outcome.classify(0, b"", b"catstack-hook-error x\n", False), "caught_error") + + def test_stderr_hook_error_requires_line_start(self): + self.assertEqual(outcome.classify(0, b"", b"x catstack-hook-error y\n", False), "silent") + + def test_json_decision_block_blocks(self): + self.assertEqual(outcome.classify(0, b'{"decision":"block"}', b"", False), "blocked") + + def test_json_continue_false_blocks(self): + self.assertEqual(outcome.classify(0, b'{"continue":false}', b"", False), "blocked") + + def test_json_permission_decision_deny_blocks(self): + data = b'{"hookSpecificOutput":{"permissionDecision":"deny"}}' + self.assertEqual(outcome.classify(0, data, b"", False), "blocked") + + def test_json_permission_deny_blocks(self): + self.assertEqual(outcome.classify(0, b'{"permission":"deny"}', b"", False), "blocked") + + def test_non_json_stdout_speaks(self): + self.assertEqual(outcome.classify(0, b"{not json", b"", False), "spoke") + + def test_json_array_stdout_speaks(self): + self.assertEqual(outcome.classify(0, b"[1]", b"", False), "spoke") + + def test_whitespace_stdout_is_silent(self): + self.assertEqual(outcome.classify(0, b" \n\t", b"", False), "silent") + + def test_empty_stdout_is_silent(self): + self.assertEqual(outcome.classify(0, b"", b"", False), "silent") + + +if __name__ == "__main__": + unittest.main() diff --git a/engine/hooks/_runner/tests/test_run.py b/engine/hooks/_runner/tests/test_run.py new file mode 100644 index 00000000..7fbc4893 --- /dev/null +++ b/engine/hooks/_runner/tests/test_run.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest + +RUNNER_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +class RunnerCLI(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.home = os.path.join(self.tmp.name, "home") + self.hooks_root = os.path.join(self.home, ".claude", "hooks") + self.runner_dir = os.path.join(self.hooks_root, "_runner") + self.fixture_dir = os.path.join(self.hooks_root, "fixture") + self.metrics_dir = os.path.join(self.tmp.name, "metrics") + os.makedirs(self.runner_dir) + os.makedirs(self.fixture_dir) + shutil.copy2(os.path.join(RUNNER_DIR, "run.py"), os.path.join(self.runner_dir, "run.py")) + shutil.copy2(os.path.join(RUNNER_DIR, "outcome.py"), os.path.join(self.runner_dir, "outcome.py")) + self._write_fixture("silent.py", "") + self._write_fixture("spoke.py", "import json\nprint(json.dumps({'hookSpecificOutput': {'additionalContext': 'hi'}}))\n") + self._write_fixture("block_exit2.py", "import sys\nsys.stderr.write('blocked\\n')\nsys.exit(2)\n") + self._write_fixture("block_json.py", "print('{\"decision\":\"block\",\"reason\":\"x\"}')\n") + self._write_fixture("crash.py", "raise RuntimeError('boom')\n") + self._write_fixture("slow.py", "import time\ntime.sleep(5)\n") + self._write_fixture("caught.py", "import sys\nsys.stderr.write('catstack-hook-error fixture: ValueError: x\\n')\n") + + def _write_fixture(self, name: str, body: str) -> None: + with open(os.path.join(self.fixture_dir, name), "w", encoding="utf-8") as handle: + handle.write(body) + + def _env(self, metrics_dir: str | None = None) -> dict[str, str]: + env = os.environ.copy() + env["CATSTACK_HOOK_METRICS_DIR"] = self.metrics_dir if metrics_dir is None else metrics_dir + return env + + def _stdin(self) -> bytes: + return json.dumps({"hook_event_name": "PromptSubmit", "session_id": "s1"}).encode() + + def _direct(self, script: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [sys.executable, os.path.join(self.fixture_dir, script)], + input=self._stdin(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env(), + ) + + def _runner(self, script: str, *args: str, metrics_dir: str | None = None) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [sys.executable, os.path.join(self.runner_dir, "run.py"), *args, f"fixture/{script}"], + input=self._stdin(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env(metrics_dir), + ) + + def _row(self) -> dict[str, object]: + with open(os.path.join(self.metrics_dir, "runs.jsonl"), encoding="utf-8") as handle: + rows = [json.loads(line) for line in handle] + self.assertEqual(len(rows), 1) + return rows[0] + + def _assert_run_matches_direct(self, script: str, outcome: str) -> None: + direct = self._direct(script) + wrapped = self._runner(script) + self.assertEqual(wrapped.stdout, direct.stdout) + self.assertEqual(wrapped.stderr, direct.stderr) + self.assertEqual(wrapped.returncode, direct.returncode) + row = self._row() + self.assertEqual(row["outcome"], outcome) + self.assertEqual(row["harness"], "claude") + self.assertEqual(row["hook"], "fixture") + self.assertEqual(row["script"], script) + self.assertEqual(row["event"], "PromptSubmit") + self.assertEqual(row["session_id"], "s1") + self.assertEqual(row["exit_code"], direct.returncode) + self.assertEqual(row["stdout_bytes"], len(direct.stdout)) + + def test_silent_hook_stays_silent(self): + self._assert_run_matches_direct("silent.py", "silent") + + def test_spoke_hook_keeps_stdout_bytes(self): + self._assert_run_matches_direct("spoke.py", "spoke") + + def test_exit_two_hook_blocks(self): + self._assert_run_matches_direct("block_exit2.py", "blocked") + + def test_block_json_hook_blocks(self): + self._assert_run_matches_direct("block_json.py", "blocked") + + def test_crash_hook_crashes(self): + self._assert_run_matches_direct("crash.py", "crashed") + + def test_caught_error_hook_is_caught(self): + self._assert_run_matches_direct("caught.py", "caught_error") + + def test_slow_hook_times_out(self): + wrapped = self._runner("slow.py", "--timeout", "1") + self.assertEqual(wrapped.stdout, b"") + self.assertIn(b"catstack-hook-runner: fixture/slow.py timed out after 1s\n", wrapped.stderr) + self.assertEqual(wrapped.returncode, 1) + row = self._row() + self.assertEqual(row["outcome"], "timed_out") + self.assertEqual(row["harness"], "claude") + self.assertEqual(row["hook"], "fixture") + self.assertEqual(row["script"], "slow.py") + self.assertEqual(row["event"], "PromptSubmit") + self.assertEqual(row["exit_code"], 1) + + def test_missing_hook_script_records_crash(self): + wrapped = self._runner("missing.py") + self.assertEqual(wrapped.stdout, b"") + self.assertEqual(wrapped.returncode, 1) + self.assertIn(b"catstack-hook-runner: no such hook script:", wrapped.stderr) + row = self._row() + self.assertEqual(row["outcome"], "crashed") + self.assertEqual(row["hook"], "fixture") + self.assertEqual(row["script"], "missing.py") + self.assertEqual(row["exit_code"], 1) + + def test_metrics_write_failure_adds_one_stderr_line(self): + direct = self._direct("spoke.py") + metrics_file = os.path.join(self.tmp.name, "metrics-file") + with open(metrics_file, "w", encoding="utf-8") as handle: + handle.write("") + wrapped = self._runner("spoke.py", metrics_dir=metrics_file) + self.assertEqual(wrapped.stdout, direct.stdout) + self.assertEqual(wrapped.returncode, direct.returncode) + self.assertEqual(direct.stderr, b"") + self.assertIn(b"catstack-hook-metrics: could not write row", wrapped.stderr) + self.assertEqual(len([line for line in wrapped.stderr.splitlines() if line]), 1) + + def test_non_json_stdin_records_null_event_and_conversation_id_fallback(self): + wrapped = subprocess.run( + [sys.executable, os.path.join(self.runner_dir, "run.py"), "fixture/silent.py"], + input=b"not json", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env(), + ) + self.assertEqual(wrapped.returncode, 0) + row = self._row() + self.assertIsNone(row["event"]) + self.assertIsNone(row["session_id"]) + + def test_conversation_id_records_session_id_when_session_id_absent(self): + wrapped = subprocess.run( + [sys.executable, os.path.join(self.runner_dir, "run.py"), "fixture/silent.py"], + input=json.dumps({"hook_event_name": "Stop", "conversation_id": "c1"}).encode(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=self._env(), + ) + self.assertEqual(wrapped.returncode, 0) + row = self._row() + self.assertEqual(row["event"], "Stop") + self.assertEqual(row["session_id"], "c1") + + +if __name__ == "__main__": + unittest.main() From 0ff3ce7ff3b97cd83d94cf44aecba751ac140775 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sat, 12 Sep 2026 19:26:54 +0000 Subject: [PATCH 2/7] =?UTF-8?q?invoker:=20wf-1789240290960-24/add-hook-run?= =?UTF-8?q?ner=20=E2=80=94=20Review=20claim:=20engine/hooks/=5Frunner/run.?= =?UTF-8?q?py=20runs=20one=20hook=20script,=20passes=20its=20stdout,=20std?= =?UTF-8?q?err=20and=20exit=20code=20through=20unchanged,=20and=20appends?= =?UTF-8?q?=20one=20metrics=20row=20with=20a=20classified=20outcome.=20Rev?= =?UTF-8?q?iew=20lane:=20behavior=20Safety=20invariant:=20A=20hook=20run?= =?UTF-8?q?=20through=20the=20runner=20produces=20byte-identical=20stdout?= =?UTF-8?q?=20and=20the=20same=20exit=20code=20as=20running=20it=20directl?= =?UTF-8?q?y;=20a=20failed=20metrics=20write=20changes=20neither=20and=20a?= =?UTF-8?q?dds=20one=20stderr=20line.=20Effectiveness=20measurement:=20Fix?= =?UTF-8?q?ture=20hooks=20run=20directly=20and=20through=20the=20runner=20?= =?UTF-8?q?give=20identical=20stdout=20bytes=20and=20exit=20codes,=20and?= =?UTF-8?q?=20each=20row=20carries=20the=20expected=20outcome.=20Slice=20r?= =?UTF-8?q?ationale:=20The=20runner=20and=20its=20outcome=20rules=20are=20?= =?UTF-8?q?one=20claim,=20reviewable=20before=20any=20install=20wiring.=20?= =?UTF-8?q?Architectural=20effect:=20Adds=20a=20shared,=20harness-agnostic?= =?UTF-8?q?=20hook=20runner=20under=20engine/hooks/=5Frunner/=20next=20to?= =?UTF-8?q?=20engine/hooks/=5Fmarkers/.=20Dormant=20until=20installed.=20G?= =?UTF-8?q?oal:=20Create=20the=20runner,=20the=20pure=20outcome=20classifi?= =?UTF-8?q?er,=20and=20tests.=20Motivation:=20No=20record=20exists=20today?= =?UTF-8?q?=20of=20which=20hooks=20fire,=20stay=20silent,=20or=20crash.=20?= =?UTF-8?q?Alternative=20considerations:=20Editing=20all=2082=20entrypoint?= =?UTF-8?q?s=20to=20import=20a=20logging=20decorator=20was=20rejected:=20i?= =?UTF-8?q?t=20cannot=20record=20import=20errors,=20syntax=20errors,=20or?= =?UTF-8?q?=20timeouts,=20and=20touches=20every=20hook.=20An=20in-process?= =?UTF-8?q?=20runpy=20runner=20was=20rejected=20because=20a=20harness-kill?= =?UTF-8?q?ed=20or=20hanging=20hook=20would=20take=20the=20recorder=20down?= =?UTF-8?q?=20with=20it.=20Implementation=20details:=20A=20subprocess=20wr?= =?UTF-8?q?apper=20plus=20a=20pure=20classifier=20in=20outcome.py.=20Non-g?= =?UTF-8?q?oals:=20No=20install.sh,=20settings,=20or=20hook=20fragment=20c?= =?UTF-8?q?hange;=20no=20report=20CLI;=20no=20change=20to=20any=20existing?= =?UTF-8?q?=20hook.=20Layer:=20domain=20Feature=20state:=20dormant=20Files?= =?UTF-8?q?:=20-=20engine/hooks/=5Frunner/run.py=20-=20engine/hooks/=5Frun?= =?UTF-8?q?ner/outcome.py=20-=20engine/hooks/=5Frunner/tests/test=5Foutcom?= =?UTF-8?q?e.py=20-=20engine/hooks/=5Frunner/tests/test=5Frun.py=20-=20eng?= =?UTF-8?q?ine/hooks/=5Frunner/tests/fixtures/=20Change=20types:=20-=20eng?= =?UTF-8?q?ine/hooks/=5Frunner/run.py:=20create=20-=20engine/hooks/=5Frunn?= =?UTF-8?q?er/outcome.py:=20create=20-=20engine/hooks/=5Frunner/tests/test?= =?UTF-8?q?=5Foutcome.py:=20create=20-=20engine/hooks/=5Frunner/tests/test?= =?UTF-8?q?=5Frun.py:=20create=20-=20engine/hooks/=5Frunner/tests/fixtures?= =?UTF-8?q?/:=20create=20Acceptance=20criteria:=20-=20`python3=20-m=20unit?= =?UTF-8?q?test=20discover=20-s=20engine/hooks/=5Frunner/tests=20-v`=20exi?= =?UTF-8?q?ts=200.=20-=20`bash=20scripts/run=5Fall=5Ftests.sh`=20exits=200?= =?UTF-8?q?.=20-=20`python3=20scripts/check=5Fhook=5Ftest=5Fcoverage.py`?= =?UTF-8?q?=20exits=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: e5ba51e8-de9e-430f-8672-e0eef3e3bafc From 5d4815896c26efc24416883982aa3e50944cd116 Mon Sep 17 00:00:00 2001 From: Invoker Date: Sat, 12 Sep 2026 19:28:02 +0000 Subject: [PATCH 3/7] Describe hook runner metrics --- engine/hooks/_runner/README.md | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 engine/hooks/_runner/README.md diff --git a/engine/hooks/_runner/README.md b/engine/hooks/_runner/README.md new file mode 100644 index 00000000..92f705de --- /dev/null +++ b/engine/hooks/_runner/README.md @@ -0,0 +1,48 @@ +# Hook runner + +Usage: + +```sh +python3 engine/hooks/_runner/run.py [--timeout SECONDS] / [args...] +``` + +The runner reads stdin, runs the hook script in a subprocess with that stdin, +passes through the hook's stdout, stderr, and exit code, then appends one JSONL +metrics row. + +Rows are written to `~/.cache/catstack-hook-metrics/runs.jsonl` by default. Set +`CATSTACK_HOOK_METRICS_DIR` to write `runs.jsonl` under a different directory. + +Each row contains: + +- `ts`: UTC timestamp for the recorded run. +- `harness`: `claude`, `cursor`, `codex`, or `unknown`, inferred from the hooks path. +- `hook`: the first path segment from `/`. +- `script`: the rest of the hook script path after `hook`. +- `event`: `hook_event_name` from JSON stdin, or `null`. +- `session_id`: `session_id` from JSON stdin, falling back to `conversation_id`, or `null`. +- `outcome`: classified result for the run. +- `exit_code`: hook process exit code recorded by the runner. +- `duration_ms`: elapsed runner time in milliseconds. +- `stdout_bytes`: number of stdout bytes emitted by the hook. +- `stderr_tail`: final 500 decoded stderr characters, with invalid UTF-8 replaced. + +Outcome precedence is: + +1. `timed_out` when the runner timeout kills the hook. +2. `blocked` when `exit_code` is `2`. +3. `crashed` when `exit_code` is any other nonzero value. +4. `caught_error` when any stderr line starts with `catstack-hook-error `. +5. `blocked` when stdout is a JSON object with `decision: "block"`, `continue: false`, + `hookSpecificOutput.permissionDecision: "deny"`, or `permission: "deny"`. +6. `spoke` when stdout has non-whitespace bytes. +7. `silent` otherwise. + +If a metrics row cannot be written, the runner appends one stderr line after the +hook stderr: + +```text +catstack-hook-metrics: could not write row to : +``` + +Nothing calls this runner until install wiring lands. From f65a4f303ec27a4b508ff022456a57fa4b62ed9f Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sat, 12 Sep 2026 19:28:23 +0000 Subject: [PATCH 4/7] =?UTF-8?q?invoker:=20wf-1789240290960-24/describe-hoo?= =?UTF-8?q?k-runner=20=E2=80=94=20Review=20claim:=20engine/hooks/=5Frunner?= =?UTF-8?q?/README.md=20states=20what=20the=20runner=20records,=20where=20?= =?UTF-8?q?the=20rows=20go,=20and=20the=20outcome=20precedence.=20Review?= =?UTF-8?q?=20lane:=20docs=20Safety=20invariant:=20Only=20engine/hooks/=5F?= =?UTF-8?q?runner/README.md=20changes;=20no=20code,=20test,=20or=20config?= =?UTF-8?q?=20file=20is=20edited.=20Effectiveness=20measurement:=20Every?= =?UTF-8?q?=20row=20field=20and=20outcome=20named=20in=20the=20README=20ap?= =?UTF-8?q?pears=20in=20run.py=20and=20outcome.py,=20checked=20by=20readin?= =?UTF-8?q?g=20both.=20Slice=20rationale:=20Prose=20in=20its=20own=20commi?= =?UTF-8?q?t=20so=20the=20code=20commit=20stays=20one=20claim.=20Architect?= =?UTF-8?q?ural=20effect:=20None;=20prose=20only.=20Goal:=20Create=20engin?= =?UTF-8?q?e/hooks/=5Frunner/README.md.=20Motivation:=20Readers=20of=20the?= =?UTF-8?q?=20hook=20directory=20need=20the=20row=20format=20without=20rea?= =?UTF-8?q?ding=20code.=20Alternative=20considerations:=20Code=20comments?= =?UTF-8?q?=20were=20rejected;=20the=20repo=20forbids=20new=20comments.=20?= =?UTF-8?q?Implementation=20details:=20One=20new=20Markdown=20file.=20Non-?= =?UTF-8?q?goals:=20No=20code,=20test,=20or=20config=20edits.=20Layer:=20d?= =?UTF-8?q?ocs=20Feature=20state:=20dormant=20Files:=20-=20engine/hooks/?= =?UTF-8?q?=5Frunner/README.md=20Change=20types:=20-=20engine/hooks/=5Frun?= =?UTF-8?q?ner/README.md:=20create=20Acceptance=20criteria:=20-=20`test=20?= =?UTF-8?q?-f=20engine/hooks/=5Frunner/README.md`=20exits=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 917f28d4-7b67-447f-bfd1-7a26a20061da From ed0d35ec79c24811d951b238d879d813f6ce6b29 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sat, 12 Sep 2026 19:38:26 +0000 Subject: [PATCH 5/7] =?UTF-8?q?invoker:=20wf-1789240290960-24/verify-hook-?= =?UTF-8?q?runner=20=E2=80=94=20Review=20claim:=20The=20runner=20tests,=20?= =?UTF-8?q?the=20repo=20test=20suite,=20and=20the=20hook=20coverage=20gate?= =?UTF-8?q?=20pass.=20Review=20lane:=20proof=20Safety=20invariant:=20Verif?= =?UTF-8?q?ication=20is=20read-only=20and=20does=20not=20alter=20any=20rep?= =?UTF-8?q?ository=20file.=20Effectiveness=20measurement:=20The=20three=20?= =?UTF-8?q?commands=20are=20the=20direct=20measurement.=20Slice=20rational?= =?UTF-8?q?e:=20One=20focused=20proof=20before=20review.=20Architectural?= =?UTF-8?q?=20effect:=20None;=20verification=20only.=20Goal:=20Prove=20pas?= =?UTF-8?q?s-through=20and=20outcome=20classification.=20Motivation:=20Run?= =?UTF-8?q?ning=20the=20tests=20is=20the=20proof.=20Alternative=20consider?= =?UTF-8?q?ations:=20A=20live-harness=20run=20is=20deferred=20to=20step=20?= =?UTF-8?q?2,=20where=20the=20runner=20is=20installed.=20Implementation=20?= =?UTF-8?q?details:=20Run=20the=20three=20commands.=20Non-goals:=20No=20mu?= =?UTF-8?q?tations.=20Layer:=20app=5Fregression=20Feature=20state:=20activ?= =?UTF-8?q?e=20Acceptance=20criteria:=20-=20Exits=200=20only=20when=20all?= =?UTF-8?q?=20pass.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: e73029a9-e2ea-4d00-8c66-f727763900a1 From 6d256c4fdc58403652993486920efd69faf861d5 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sat, 12 Sep 2026 19:41:32 +0000 Subject: [PATCH 6/7] =?UTF-8?q?invoker:=20wf-1789240290960-24/scrub-handof?= =?UTF-8?q?f-artifacts=20=E2=80=94=20Review=20claim:=20No=20ephemeral=20in?= =?UTF-8?q?ter-task=20handoff=20files=20remain=20in=20the=20worktree=20bef?= =?UTF-8?q?ore=20the=20merge=20gate.=20Review=20lane:=20cleanup=20Safety?= =?UTF-8?q?=20invariant:=20The=20scrub=20script=20only=20checks=20for=20kn?= =?UTF-8?q?own=20handoff=20artifact=20names=20and=20never=20touches=20sour?= =?UTF-8?q?ce,=20tests,=20or=20other=20repository=20files.=20Effectiveness?= =?UTF-8?q?=20measurement:=20The=20script=20exits=20non-zero=20if=20any=20?= =?UTF-8?q?handoff=20artifact=20remains.=20Slice=20rationale:=20Required?= =?UTF-8?q?=20terminal=20scrub=20for=20every=20implementation=20workflow.?= =?UTF-8?q?=20Architectural=20effect:=20None;=20hygiene=20only.=20Goal:=20?= =?UTF-8?q?Leave=20the=20branch=20free=20of=20handoff=20artifacts.=20Motiv?= =?UTF-8?q?ation:=20Handoff=20files=20must=20not=20reach=20the=20PR.=20Alt?= =?UTF-8?q?ernative=20considerations:=20Manual=20cleanup=20was=20rejected?= =?UTF-8?q?=20as=20non-deterministic.=20Implementation=20details:=20Run=20?= =?UTF-8?q?scripts/scrub-handoff-artifacts.sh.=20Layer=20exception:=20allo?= =?UTF-8?q?wed=20--=20the=20terminal=20scrub=20must=20run=20after=20every?= =?UTF-8?q?=20task=20in=20the=20workflow,=20including=20the=20docs=20task.?= =?UTF-8?q?=20Non-goals:=20No=20product=20edits.=20Layer:=20app=5Fregressi?= =?UTF-8?q?on=20Feature=20state:=20active=20Acceptance=20criteria:=20-=20`?= =?UTF-8?q?bash=20scripts/scrub-handoff-artifacts.sh`=20exits=200.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: cb09f6f2-66d1-4564-adb3-38e63423dbfa From 6a832645529337c24259963a167619c89e3596ed Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sat, 12 Sep 2026 12:56:15 -0700 Subject: [PATCH 7/7] fix(hooks/_runner): run on Python 3.9 and forward the hook result when recording fails datetime.UTC exists only on 3.11+, so on CI's Python 3.9 the runner raised after the hook ran and dropped the hook's stdout and exit code. Use datetime.timezone.utc, and catch any failure while classifying or writing the metrics row so the hook's output is still forwarded, with one stderr line naming the error. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01G95BG4NxDsW4NA6fcipHrv Change-Id: Ie823ca268e85e83dd57d3b6f1b026946daeb4c99 --- engine/hooks/_runner/run.py | 11 +++++++---- engine/hooks/_runner/tests/test_run.py | 9 +++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/engine/hooks/_runner/run.py b/engine/hooks/_runner/run.py index 1cdf2776..bcc2fd7a 100644 --- a/engine/hooks/_runner/run.py +++ b/engine/hooks/_runner/run.py @@ -81,7 +81,7 @@ def _row( ) -> dict[str, object]: event, session_id = _stdin_fields(stdin) return { - "ts": datetime.datetime.now(datetime.UTC).isoformat(), + "ts": datetime.datetime.now(datetime.timezone.utc).isoformat(), "harness": _harness(hooks_root), "hook": hook, "script": script, @@ -132,9 +132,12 @@ def main(argv: list[str] | None = None) -> int: ).encode() exit_code = 1 - outcome = classify(exit_code, stdout, stderr, timed_out) - row = _row(hooks_root, hook, script, stdin, outcome, exit_code, started, stdout, stderr) - metrics_error = _write_metrics(row, _metrics_path()) + try: + outcome = classify(exit_code, stdout, stderr, timed_out) + row = _row(hooks_root, hook, script, stdin, outcome, exit_code, started, stdout, stderr) + metrics_error = _write_metrics(row, _metrics_path()) + except Exception as exc: + metrics_error = f"catstack-hook-metrics: could not record run: {type(exc).__name__}: {exc}\n".encode() sys.stdout.buffer.write(stdout) sys.stdout.buffer.flush() sys.stderr.buffer.write(stderr) diff --git a/engine/hooks/_runner/tests/test_run.py b/engine/hooks/_runner/tests/test_run.py index 7fbc4893..cc5b2186 100644 --- a/engine/hooks/_runner/tests/test_run.py +++ b/engine/hooks/_runner/tests/test_run.py @@ -68,6 +68,15 @@ def _row(self) -> dict[str, object]: self.assertEqual(len(rows), 1) return rows[0] + def test_recording_failure_still_forwards_hook_result(self) -> None: + with open(os.path.join(self.runner_dir, "outcome.py"), "w", encoding="utf-8") as handle: + handle.write("def classify(*args, **kwargs):\n raise RuntimeError('classify broke')\n") + direct = self._direct("spoke.py") + wrapped = self._runner("spoke.py") + self.assertEqual(wrapped.stdout, direct.stdout) + self.assertEqual(wrapped.returncode, direct.returncode) + self.assertIn(b"catstack-hook-metrics: could not record run: RuntimeError: classify broke", wrapped.stderr) + def _assert_run_matches_direct(self, script: str, outcome: str) -> None: direct = self._direct(script) wrapped = self._runner(script)