fix(ci): report an eval policy probe with no tool call as unproven

.github/actions/resolve-config/callers.test.mjs

@@ -9633,8 +9633,7 @@ test("every eval leg runs the same managed-policy step and the same probe", () =
     const probeOf = (s) =>
       step(s, "probe")
         .run.replaceAll("no pull request case runs", "no case runs")
-        .replace("the next leg runs the eval", "the next leg runs the pass")
-        .replace(`echo "The managed policy denied the probe's Read, Grep and Glob calls."\n`, "");
+        .replace("the next leg runs the eval", "the next leg runs the pass");
     assert.equal(probeOf(steps), probeOf(first), `${file} ${id} probe`);
     const [denyIdx, isolateIdx, probeIdx] = ["Deny the eval's reads of process and runner state", "Isolate the runner", "Probe that the policy denies those reads"].map((n) => {
       const i = steps.findIndex((s) => s.name === n);
@@ -9644,18 +9643,18 @@ test("every eval leg runs the same managed-policy step and the same probe", () =
     assert.ok(denyIdx < isolateIdx, `${file} ${id}: the policy is written while sudo works`);
     assert.ok(isolateIdx < probeIdx, `${file} ${id}`);
   }
-  const probe = step(first, "probe").run;
-  assert.match(probe, /jq -e -R -s --arg t "\$1" --arg p "\$2" '\[split\("\\n"\)\[\] \| fromjson\? \| objects\] \| last\(/);
-  assert.doesNotMatch(probe, /jq -e -s /, "a slurp fails on any line that is not JSON");
-  assert.match(probe, /so no case runs \(probe: \$\(probe_facts\)\)"; exit 1; \}/);
+  // policy-probe.mjs reads the stream line by line and names the probe's closed facts.
+  assert.match(step(first, "probe").run, /PROBE="\$out" node \.base\/\.github\/scripts\/plugin-eval\/policy-probe\.mjs\) \|\| judged=\$\?\n/);
 });
 
 // The model can decline the probe's reads and end with no tool call,
 // which proves nothing about the policy: that session runs again,
-// three times at most. Any other shortfall stops the job on the first session.
+// three times at most, and then stops the job as unproven, never as a policy
+// failure. A call made and not denied stops the job on the first session.
 // Runs the step with a stub claude that replays one hand-written stream per
-// call and a stub node standing in for the handoff classifier.
-test("the policy probe reruns only a session with no tool call, at most three times", () => {
+// call, a stub node standing in for the handoff classifier, and the real
+// policy-probe.mjs.
+test("the policy probe reruns a session with no tool call, at most three times, then stops as unproven", () => {
   const step = workflow("eval-hill-climb-pass.yml").jobs.pass.steps.find((s) => s.id === "probe");
   const root = "/runner";
   const calls = [
@@ -9682,7 +9681,8 @@ test("the policy probe reruns only a session with no tool call, at most three ti
       streams.forEach((s, i) => writeFileSync(join(dir, `stream-${i + 1}`), `${s}\n`));
       const stub = (name, body) => writeFileSync(join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 });
       stub("claude", `n=$(($(cat "${dir}/calls" 2>/dev/null || echo 0) + 1)); echo "$n" >"${dir}/calls"\ncat "${dir}/stream-$n"`);
-      stub("node", "printf 'handoff=false\\nhandoff-reason=\\nlane-class=\\noutcome=success\\n'");
+      const judge = fileURLToPath(new URL("../../scripts/plugin-eval/policy-probe.mjs", import.meta.url));
+      stub("node", `case "$1" in *policy-probe.mjs) exec "${process.execPath}" "${judge}" ;; esac\nprintf 'handoff=false\\nhandoff-reason=\\nlane-class=\\noutcome=success\\n'`);
       const output = join(dir, "output");
       writeFileSync(output, "");
       const result = spawnSync("bash", ["-e", "-c", step.run], {
@@ -9699,13 +9699,14 @@ test("the policy probe reruns only a session with no tool call, at most three ti
   assert.deepEqual([first.status, first.calls], [0, 1], first.stdout);
   const retried = probe([declined, allDenied]);
   assert.deepEqual([retried.status, retried.calls], [0, 2], retried.stdout);
-  assert.match(retried.stdout, /^::notice::the probe's session made no tool call, so it proved nothing about the policy \(attempt 1 of 3\)$/m);
+  assert.match(retried.stdout, /^::notice::the probe's session made no tool call, so it proved nothing about the policy \(probe: .* tool_calls=none denials=0 .*\) \(attempt 1 of 3\)$/m);
   const never = probe([declined, declined, declined, allDenied]);
   assert.deepEqual([never.status, never.calls], [1, 3], never.stdout);
-  assert.match(never.stdout, /^::error::the managed policy did not record a denied Read of \/proc\/self\/environ, so no case runs \(probe: .* tool_calls=none denials=0 /m);
+  assert.match(never.stdout, /^::error::the policy is unproven: in 3 attempts the probe's session never made every call it was asked for, so no case runs \(last attempt: the probe's session made no tool call, .* tool_calls=none denials=0 /m);
+  assert.doesNotMatch(never.stdout, /did not record a denied/);
   const undenied = probe([readNotDenied, allDenied]);
   assert.deepEqual([undenied.status, undenied.calls], [1, 1], undenied.stdout);
-  assert.match(undenied.stdout, /^::error::the managed policy did not record a denied Read of \/proc\/self\/environ, so no case runs \(probe: .* tool_calls=Read denials=0 /m);
+  assert.match(undenied.stdout, /^::error::the managed policy did not record a denied Read of \/proc\/self\/environ \(probe: .* tool_calls=Read denials=0 .*\), so no case runs$/m);
 });
 
 // --- mutated code never runs beside a credential (#13586) ---

.github/scripts/plugin-eval/policy-probe.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+// policy-probe.mjs: judges the eval policy probe's stream (`claude -p
+// --output-format stream-json --verbose`) for the four calls the eval legs'
+// probe step asks for. Each call is denied (the result's permission_denials
+// records it), made and not denied (a tool_use with no matching denial: the
+// policy let it through), or not made. A session can end normally without
+// making every call, since no `claude` flag forces a tool call; that proves
+// nothing about the policy either way, so it is reported as unproven, never as
+// a policy failure.
+//
+// Env: PROBE (the stream's path), RUNNER_ROOT.
+// Exit 0: all four denied. Exit 1: a call made and not denied, or a session
+// that errored before making every call. Exit 3: a session that ended normally
+// without making every call, which the caller runs again.
+// Prints one line: the reason, then the probe's facts. The facts hold only
+// counts and tokens checked against fixed patterns, never text from the
+// stream, which can hold what a call returned.
+import { readFileSync } from "node:fs";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
+
+export const expectedCalls = (runnerRoot) => [
+  ["Read", "/proc/self/environ"],
+  ["Read", `${runnerRoot}/.runner`],
+  ["Grep", "/proc/self"],
+  ["Glob", "/proc/self"],
+];
+
+const pathOf = (input) => String(input?.file_path ?? input?.path ?? "");
+const token = (value, pattern) => (pattern.test(String(value)) ? String(value) : "other");
+
+export function judgeProbe(text, runnerRoot) {
+  const lines = text.split("\n").filter((line) => line !== "");
+  const messages = lines.flatMap((line) => {
+    try {
+      const value = JSON.parse(line);
+      return value && typeof value === "object" && !Array.isArray(value) ? [value] : [];
+    } catch {
+      return [];
+    }
+  });
+  const result = messages.filter((m) => m.type === "result").at(-1) ?? null;
+  const uses = messages
+    .filter((m) => m.type === "assistant")
+    .flatMap((m) => (Array.isArray(m.message?.content) ? m.message.content : []))
+    .filter((block) => block && typeof block === "object" && block.type === "tool_use");
+  const denials = Array.isArray(result?.permission_denials) ? result.permission_denials : [];
+
+  const facts = [
+    `result=${result ? "present" : "none"}`,
+    `subtype=${token(result?.subtype ?? "none", /^[a-z_]{1,40}$/)}`,
+    `terminal_reason=${token(result?.terminal_reason ?? "none", /^[a-z_]{1,40}$/)}`,
+    `is_error=${token(result?.is_error ?? "none", /^(true|false|none)$/)}`,
+    `api_error_status=${token(result?.api_error_status ?? "none", /^([0-9]{3}|none)$/)}`,
+    `num_turns=${token(result?.num_turns ?? "none", /^([0-9]{1,4}|none)$/)}`,
+    `tool_calls=${uses.length ? uses.map((u) => (["Read", "Grep", "Glob"].includes(u.name) ? u.name : "other")).join(",") : "none"}`,
+    `denials=${denials.length}`,
+    `unparsable_lines=${lines.length - messages.length}`,
+  ].join(" ");
+
+  const calls = expectedCalls(runnerRoot).map(([tool, prefix]) => ({
+    tool,
+    prefix,
+    denied: denials.some((d) => d?.tool_name === tool && pathOf(d.tool_input).startsWith(prefix)),
+    made: uses.some((u) => u.name === tool && pathOf(u.input).startsWith(prefix)),
+  }));
+  const say = (code, reason) => ({ code, line: `${reason} (probe: ${facts})` });
+
+  if (calls.every((c) => c.denied)) return say(0, "the managed policy denied the probe's Read, Grep and Glob calls");
+  const leaked = calls.find((c) => c.made && !c.denied);
+  if (leaked) return say(1, `the managed policy did not record a denied ${leaked.tool} of ${leaked.prefix}`);
+  const missing = calls.find((c) => !c.denied);
+  if (result?.subtype !== "success") {
+    return say(1, `the probe's session ended in error before its ${missing.tool} of ${missing.prefix}, so it proved nothing about the policy`);
+  }
+  if (uses.length === 0) return say(3, "the probe's session made no tool call, so it proved nothing about the policy");
+  return say(3, `the probe's session never made its ${missing.tool} of ${missing.prefix}, so it proved nothing about the policy`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  let text = "";
+  try {
+    text = readFileSync(process.env.PROBE ?? "", "utf8");
+  } catch {
+    // An unreadable stream judges as an empty one: no result, so an error.
+  }
+  const { code, line } = judgeProbe(text, process.env.RUNNER_ROOT ?? "");
+  process.stdout.write(`${line}\n`);
+  process.exitCode = code;
+}

.github/scripts/plugin-eval/policy-probe.test.mjs

@@ -0,0 +1,72 @@
+// policy-probe.mjs proves the managed policy denied the probe's four reads,
+// and a session that never makes them is unproven, never a policy failure:
+// the workflow step stops the leg on both, so no case runs on a policy nobody
+// proved. The stream shapes are stream-json's: tool_use blocks in assistant
+// messages and the result's permission_denials (code.claude.com/docs/en/
+// agent-sdk/typescript, "SDKResultMessage"). The step itself runs end to end
+// in callers.test.mjs.
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { judgeProbe } from "./policy-probe.mjs";
+
+const ROOT = "/home/runner/actions-runner";
+const CALLS = [
+  ["Read", { file_path: "/proc/self/environ" }],
+  ["Read", { file_path: `${ROOT}/.runner` }],
+  ["Grep", { pattern: "policy-probe", path: "/proc/self/environ" }],
+  ["Glob", { pattern: "environ", path: "/proc/self" }],
+];
+const stream = ({ calls = CALLS, denied = calls, subtype = "success" } = {}) =>
+  [
+    { type: "system", subtype: "init" },
+    ...calls.map(([name, input]) => ({ type: "assistant", message: { content: [{ type: "tool_use", name, input }] } })),
+    { type: "assistant", message: { content: [{ type: "text", text: "done" }] } },
+    { type: "result", subtype, is_error: subtype !== "success", num_turns: 2, permission_denials: denied.map(([tool_name, tool_input]) => ({ tool_name, tool_input })) },
+  ]
+    .map((m) => JSON.stringify(m))
+    .join("\n");
+
+test("all four calls denied proves the policy", () => {
+  const { code, line } = judgeProbe(stream(), ROOT);
+  assert.equal(code, 0);
+  assert.match(line, /^the managed policy denied the probe's Read, Grep and Glob calls \(probe: .* tool_calls=Read,Read,Grep,Glob denials=4 /);
+});
+
+test("a session with no tool call is unproven and runs again, never a policy failure", () => {
+  const { code, line } = judgeProbe(stream({ calls: [] }), ROOT);
+  assert.equal(code, 3);
+  assert.match(line, /^the probe's session made no tool call, so it proved nothing about the policy \(probe: .*tool_calls=none denials=0/);
+  assert.doesNotMatch(line, /did not record/);
+});
+
+test("a session that skips a call is unproven, naming the call", () => {
+  const { code, line } = judgeProbe(stream({ calls: CALLS.slice(0, 3) }), ROOT);
+  assert.equal(code, 3);
+  assert.match(line, /^the probe's session never made its Glob of \/proc\/self, so it proved nothing/);
+});
+
+test("a call made and not denied is a policy failure", () => {
+  const { code, line } = judgeProbe(stream({ denied: [CALLS[0], CALLS[1], CALLS[3]] }), ROOT);
+  assert.equal(code, 1);
+  assert.match(line, /^the managed policy did not record a denied Grep of \/proc\/self \(probe: /);
+});
+
+test("a session that errored before its calls is its own error, not a policy failure", () => {
+  for (const text of [stream({ calls: [], subtype: "error_max_turns" }), "", "not json\n"]) {
+    const { code, line } = judgeProbe(text, ROOT);
+    assert.equal(code, 1);
+    assert.match(line, /^the probe's session ended in error before its Read of \/proc\/self\/environ, so it proved nothing/);
+  }
+});
+
+test("the facts carry no text from the stream", () => {
+  const leak = JSON.stringify({ type: "result", subtype: "SECRET value", terminal_reason: "x".repeat(50), api_error_status: "sk-1", num_turns: "lots" });
+  const { line } = judgeProbe(`${leak}\n[1]\n`, ROOT);
+  assert.match(line, /subtype=other terminal_reason=other is_error=none api_error_status=other num_turns=other tool_calls=none denials=0 unparsable_lines=1\)$/);
+});
+
+test("a line that is not JSON cannot hide the denials", () => {
+  const { code, line } = judgeProbe(`not json\n${stream()}\n`, ROOT);
+  assert.equal(code, 0);
+  assert.match(line, /unparsable_lines=1\)$/);
+});

.github/scripts/plugin-eval/workflow.test.mjs

@@ -34,16 +34,16 @@ test("a pull request run proves the policy denies the reads before any case runs
   assert.match(probe, /if: steps\.stage\.outputs\.go == 'true' && github\.event_name == 'pull_request'\n/);
   assert.match(probe, /RUNNER_ROOT: \$\{\{ steps\.deny\.outputs\.runner-root \}\}/);
   assert.match(probe, /--allowedTools Read Grep Glob /);
-  assert.match(probe, /any\(\.permission_denials\[\]\?; \.tool_name == \$t and/);
-  for (const call of ["Read /proc/self/environ", 'Read "$RUNNER_ROOT/.runner"', "Grep /proc/self", "Glob /proc/self"]) {
-    assert.ok(probe.includes(`\n          denied ${call}\n`), `the probe checks a denied ${call}`);
-  }
+  // policy-probe.mjs judges the four calls; only its exit 0 lets the step pass.
+  const judge = 'reason=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" node .base/.github/scripts/plugin-eval/policy-probe.mjs) || judged=$?';
+  assert.ok(probe.includes(judge), "the judge runs from the base checkout without the credential");
+  assert.match(probe, /\n {10}case \$judged in\n {12}0\) echo "\$reason" ;;\n {12}3\) echo "::error::[^\n]*"; exit 1 ;;\n {12}\*\) echo "::error::[^\n]*"; exit 1 ;;\n {10}esac\n/);
   // The leak check looks for the leg's own credential, whichever leg runs.
   assert.match(probe, /credential=\$\{ANTHROPIC_API_KEY:-\$\{CLAUDE_CODE_OAUTH_TOKEN:-\}\}\n {10}\[ -n "\$credential" \] \|\| /);
   assert.match(probe, /if grep -qF "\$credential" "\$out"; then\n[^\n]*\n {14}exit 1/);
   assert.match(probe, /trap 'rm -f "\$out"' EXIT/);
   // A probe the credential's own stop ended hands off before the denial checks, never past them.
-  assert.ok(probe.indexOf("eval-handoff.mjs") < probe.indexOf("denied Read /proc/self/environ"));
+  assert.ok(probe.indexOf("eval-handoff.mjs") < probe.indexOf("policy-probe.mjs"));
   assert.ok(probe.indexOf('if grep -qF "$credential"') < probe.indexOf("eval-handoff.mjs"), "the leak check runs before the classifier");
   assert.match(probe, /env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="\$out"/, "the classifier runs without the credential");
   assert.doesNotMatch(probe, /cat "\$out"|jq [^\n]*"\$out"[^\n]*\| *tee/);