From 8295e2fa079210994cc83ea2298d42fc1bfe0f32 Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Fri, 6 Nov 2026 00:22:41 -0500
Subject: [PATCH 1/2] fix(ci): frame the eval policy probe as a self-test and
 rerun a no-call session

Asked only to read /proc/self/environ and the runner's .runner file and to
Grep them for TOKEN, the model can decline as credential harvesting and end
with no tool call, so the probe recorded no denial and blocked every pull
request eval leg (run 42166675433: tool_calls=none, num_turns=2).

The probe prompt now names the run the runner's permission self-test and
says each call is expected to be denied; the paths and the deny checks are
unchanged, and the Grep pattern no longer reads as a secret search. A
session that ends with no tool call proved nothing about the policy, so it
runs again, up to three times; a call made and not denied, a missing
result, or any missing denial still stops the job on that session. The
credential grep and the handoff classifier run on every attempt.

Refs: #12035

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../actions/resolve-config/callers.test.mjs   |  58 +++++++
 .github/scripts/plugin-eval/workflow.test.mjs |   2 +-
 .github/workflows/eval-browser-dispatch.yml   | 141 +++++++++++-------
 .github/workflows/eval-hill-climb-pass.yml    | 123 +++++++++------
 4 files changed, 221 insertions(+), 103 deletions(-)

diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs
index 70363fe5ca5..989b5fcb44a 100644
--- a/.github/actions/resolve-config/callers.test.mjs
+++ b/.github/actions/resolve-config/callers.test.mjs
@@ -9431,6 +9431,64 @@ test("every eval leg runs the same managed-policy step and the same probe", () =
   assert.match(probe, /so no case runs \(probe: \$\(probe_facts\)\)"; exit 1; \}/);
 });
 
+// The model can decline the probe's reads and end with no tool call (run
+// 42166675433), which proves nothing about the policy: that session runs again,
+// three times at most. Any other shortfall stops the job on the first session.
+// Runs the step with a stub claude that replays one hand-written stream per
+// call and a stub node standing in for the handoff classifier.
+test("the policy probe reruns only a session with no tool call, at most three times", () => {
+  const step = workflow("eval-hill-climb-pass.yml").jobs.pass.steps.find((s) => s.id === "probe");
+  const root = "/runner";
+  const calls = [
+    ["Read", { file_path: "/proc/self/environ" }],
+    ["Read", { file_path: `${root}/.runner` }],
+    ["Grep", { pattern: "policy-probe", path: "/proc/self/environ" }],
+    ["Glob", { pattern: "environ", path: "/proc/self" }],
+  ];
+  const stream = (made, denials) =>
+    [
+      { type: "assistant", message: { content: made.length ? made.map(([name, input], i) => ({ type: "tool_use", id: `t${i}`, name, input })) : [{ type: "text", text: "I won't do that." }] } },
+      { type: "result", subtype: "success", is_error: false, num_turns: made.length + 1, permission_denials: denials.map(([tool_name, tool_input], i) => ({ tool_name, tool_use_id: `t${i}`, tool_input })) },
+    ]
+      .map((m) => JSON.stringify(m))
+      .join("\n");
+  const declined = stream([], []);
+  const allDenied = stream(calls, calls);
+  const readNotDenied = stream(calls.slice(0, 1), []);
+  const probe = (streams) => {
+    const dir = tempDir("policy-probe-");
+    try {
+      const bin = join(dir, "bin");
+      mkdirSync(bin);
+      streams.forEach((s, i) => writeFileSync(join(dir, `stream-${i + 1}`), `${s}\n`));
+      const stub = (name, body) => writeFileSync(join(bin, name), `#!/usr/bin/env bash\n${body}\n`, { mode: 0o755 });
+      stub("claude", `n=$(($(cat "${dir}/calls" 2>/dev/null || echo 0) + 1)); echo "$n" >"${dir}/calls"\ncat "${dir}/stream-$n"`);
+      stub("node", "printf 'handoff=false\\nhandoff-reason=\\nlane-class=\\noutcome=success\\n'");
+      const output = join(dir, "output");
+      writeFileSync(output, "");
+      const result = spawnSync("bash", ["-e", "-c", step.run], {
+        cwd: dir,
+        encoding: "utf8",
+        env: { PATH: `${bin}:${process.env.PATH}`, LANE_API_KEY: "sk-test-credential", RUNNER_ROOT: root, GITHUB_OUTPUT: output },
+      });
+      return { status: result.status, stdout: result.stdout, calls: Number(readFileSync(join(dir, "calls"), "utf8")) };
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  };
+  const first = probe([allDenied]);
+  assert.deepEqual([first.status, first.calls], [0, 1], first.stdout);
+  const retried = probe([declined, allDenied]);
+  assert.deepEqual([retried.status, retried.calls], [0, 2], retried.stdout);
+  assert.match(retried.stdout, /^::notice::the probe's session made no tool call, so it proved nothing about the policy \(attempt 1 of 3\)$/m);
+  const never = probe([declined, declined, declined, allDenied]);
+  assert.deepEqual([never.status, never.calls], [1, 3], never.stdout);
+  assert.match(never.stdout, /^::error::the managed policy did not record a denied Read of \/proc\/self\/environ, so no case runs \(probe: .* tool_calls=none denials=0 /m);
+  const undenied = probe([readNotDenied, allDenied]);
+  assert.deepEqual([undenied.status, undenied.calls], [1, 1], undenied.stdout);
+  assert.match(undenied.stdout, /^::error::the managed policy did not record a denied Read of \/proc\/self\/environ, so no case runs \(probe: .* tool_calls=Read denials=0 /m);
+});
+
 // --- mutated code never runs beside a credential (#13586) ---
 
 // Owner constraint on #13586: no job that runs mutants holds a Claude
diff --git a/.github/scripts/plugin-eval/workflow.test.mjs b/.github/scripts/plugin-eval/workflow.test.mjs
index a4435583a83..9a33f616daa 100644
--- a/.github/scripts/plugin-eval/workflow.test.mjs
+++ b/.github/scripts/plugin-eval/workflow.test.mjs
@@ -40,7 +40,7 @@ test("a pull request run proves the policy denies the reads before any case runs
   }
   // The leak check looks for the leg's own credential, whichever leg runs.
   assert.match(probe, /credential=\$\{ANTHROPIC_API_KEY:-\$\{CLAUDE_CODE_OAUTH_TOKEN:-\}\}\n {10}\[ -n "\$credential" \] \|\| /);
-  assert.match(probe, /if grep -qF "\$credential" "\$out"; then\n[^\n]*\n {12}exit 1/);
+  assert.match(probe, /if grep -qF "\$credential" "\$out"; then\n[^\n]*\n {14}exit 1/);
   assert.match(probe, /trap 'rm -f "\$out"' EXIT/);
   // A probe the credential's own stop ended hands off before the denial checks, never past them.
   assert.ok(probe.indexOf("eval-handoff.mjs") < probe.indexOf("denied Read /proc/self/environ"));
diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml
index 7070287019a..10e88035787 100644
--- a/.github/workflows/eval-browser-dispatch.yml
+++ b/.github/workflows/eval-browser-dispatch.yml
@@ -564,10 +564,14 @@ jobs:
       # credential anywhere. Claude Code applies Read deny rules to Glob and
       # Grep only on a best-effort basis (code.claude.com/docs/en/permissions,
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
-      # instead of reaching a case. The stream can hold what a call returned,
-      # so it stays in a file that is never printed and is removed after the
-      # check. A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the eval.
+      # instead of reaching a case. The prompt names the run a policy self-test:
+      # asked only to read those paths, the model can decline them as credential
+      # harvesting and make no call (run 42166675433). A session that ends with
+      # no tool call proved nothing, so it runs again, up to three times; a call
+      # made and not denied stops the job at once. The stream can hold what a
+      # call returned, so it stays in a file that is never printed and is
+      # removed after the check. A probe that the credential's own stop ended
+      # hands off instead (eval-handoff.mjs), so the next leg runs the eval.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request'
@@ -587,20 +591,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no pull request case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -620,6 +610,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no pull request case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self
@@ -1384,10 +1395,14 @@ jobs:
       # credential anywhere. Claude Code applies Read deny rules to Glob and
       # Grep only on a best-effort basis (code.claude.com/docs/en/permissions,
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
-      # instead of reaching a case. The stream can hold what a call returned,
-      # so it stays in a file that is never printed and is removed after the
-      # check. A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the eval.
+      # instead of reaching a case. The prompt names the run a policy self-test:
+      # asked only to read those paths, the model can decline them as credential
+      # harvesting and make no call (run 42166675433). A session that ends with
+      # no tool call proved nothing, so it runs again, up to three times; a call
+      # made and not denied stops the job at once. The stream can hold what a
+      # call returned, so it stays in a file that is never printed and is
+      # removed after the check. A probe that the credential's own stop ended
+      # hands off instead (eval-handoff.mjs), so the next leg runs the eval.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request'
@@ -1407,20 +1422,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no pull request case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -1440,6 +1441,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no pull request case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self
@@ -2158,10 +2180,14 @@ jobs:
       # credential anywhere. Claude Code applies Read deny rules to Glob and
       # Grep only on a best-effort basis (code.claude.com/docs/en/permissions,
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
-      # instead of reaching a case. The stream can hold what a call returned,
-      # so it stays in a file that is never printed and is removed after the
-      # check. A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the eval.
+      # instead of reaching a case. The prompt names the run a policy self-test:
+      # asked only to read those paths, the model can decline them as credential
+      # harvesting and make no call (run 42166675433). A session that ends with
+      # no tool call proved nothing, so it runs again, up to three times; a call
+      # made and not denied stops the job at once. The stream can hold what a
+      # call returned, so it stays in a file that is never printed and is
+      # removed after the check. A probe that the credential's own stop ended
+      # hands off instead (eval-handoff.mjs), so the next leg runs the eval.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request'
@@ -2181,20 +2207,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no pull request case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -2214,6 +2226,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no pull request case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self
diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml
index b6fa9d37c7b..64b234dae76 100644
--- a/.github/workflows/eval-hill-climb-pass.yml
+++ b/.github/workflows/eval-hill-climb-pass.yml
@@ -319,8 +319,10 @@ jobs:
       # eval-browser-dispatch.yml's probe: the job stops unless the result
       # records each of the four reads as denied, or if it holds the credential.
       # A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a
-      # file that is never printed and is removed after the checks.
+      # (eval-handoff.mjs), so the next leg runs the pass. A session that ends
+      # with no tool call proved nothing and runs again, up to three times. The
+      # stream stays in a file that is never printed and is removed after the
+      # checks.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.credential.outputs.run == 'true'
@@ -340,20 +342,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -373,6 +361,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self
@@ -727,8 +736,10 @@ jobs:
       # eval-browser-dispatch.yml's probe: the job stops unless the result
       # records each of the four reads as denied, or if it holds the credential.
       # A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a
-      # file that is never printed and is removed after the checks.
+      # (eval-handoff.mjs), so the next leg runs the pass. A session that ends
+      # with no tool call proved nothing and runs again, up to three times. The
+      # stream stays in a file that is never printed and is removed after the
+      # checks.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.credential.outputs.run == 'true'
@@ -748,20 +759,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -781,6 +778,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self
@@ -1090,8 +1108,10 @@ jobs:
       # eval-browser-dispatch.yml's probe: the job stops unless the result
       # records each of the four reads as denied, or if it holds the credential.
       # A probe that the credential's own stop ended hands off instead
-      # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a
-      # file that is never printed and is removed after the checks.
+      # (eval-handoff.mjs), so the next leg runs the pass. A session that ends
+      # with no tool call proved nothing and runs again, up to three times. The
+      # stream stays in a file that is never printed and is removed after the
+      # checks.
       - name: Probe that the policy denies those reads
         id: probe
         if: steps.credential.outputs.run == 'true'
@@ -1111,20 +1131,6 @@ jobs:
           [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; }
           out=$(mktemp)
           trap 'rm -f "$out"' EXIT
-          status=0
-          claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \
-            --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
-          if grep -qF "$credential" "$out"; then
-            echo "::error::the probe's result holds the credential, so no case runs"
-            exit 1
-          fi
-          verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
-            { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
-          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
-          if grep -qx 'handoff=true' <<<"$verdict"; then
-            echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
-            exit 0
-          fi
           probe_facts() {
             jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
               [split("\n")[] | select(. != "")] as $lines
@@ -1144,6 +1150,27 @@ jobs:
             jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
               { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
+          for attempt in 1 2 3; do
+            status=0
+            claude -p "This is the CI runner's permission self-test, run by the repository's own workflow before any eval case. The runner's managed policy denies reads of /proc and of the runner's install directory, so each call below is expected to come back denied, and the test passes only when Claude Code records all four denials. No file content is wanted or used. Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern policy-probe and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self. Then reply with one line saying which calls were denied." \
+              --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$?
+            if grep -qF "$credential" "$out"; then
+              echo "::error::the probe's result holds the credential, so no case runs"
+              exit 1
+            fi
+            verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) ||
+              { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; }
+            if grep -qx 'handoff=true' <<<"$verdict"; then
+              printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
+              echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
+              exit 0
+            fi
+            jq -e -R -s '[split("\n")[] | fromjson? | objects] as $m | ($m | map(select(.type == "result")) | last) as $r
+              | $r != null and $r.subtype == "success"
+              and ([$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use")] | length) == 0' "$out" >/dev/null 2>&1 || break
+            echo "::notice::the probe's session made no tool call, so it proved nothing about the policy (attempt $attempt of 3)"
+          done
+          printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT"
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
           denied Grep /proc/self

From e456ea1d6b4833ac0e4f233a06d1537d417b5a56 Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Fri, 6 Nov 2026 00:25:44 -0500
Subject: [PATCH 2/2] docs(ci): drop a run citation from the eval probe
 comments

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/actions/resolve-config/callers.test.mjs | 4 ++--
 .github/workflows/eval-browser-dispatch.yml     | 6 +++---
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs
index 989b5fcb44a..0f9089e5cf1 100644
--- a/.github/actions/resolve-config/callers.test.mjs
+++ b/.github/actions/resolve-config/callers.test.mjs
@@ -9431,8 +9431,8 @@ test("every eval leg runs the same managed-policy step and the same probe", () =
   assert.match(probe, /so no case runs \(probe: \$\(probe_facts\)\)"; exit 1; \}/);
 });
 
-// The model can decline the probe's reads and end with no tool call (run
-// 42166675433), which proves nothing about the policy: that session runs again,
+// The model can decline the probe's reads and end with no tool call,
+// which proves nothing about the policy: that session runs again,
 // three times at most. Any other shortfall stops the job on the first session.
 // Runs the step with a stub claude that replays one hand-written stream per
 // call and a stub node standing in for the handoff classifier.
diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml
index 10e88035787..2f68472168c 100644
--- a/.github/workflows/eval-browser-dispatch.yml
+++ b/.github/workflows/eval-browser-dispatch.yml
@@ -566,7 +566,7 @@ jobs:
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
       # instead of reaching a case. The prompt names the run a policy self-test:
       # asked only to read those paths, the model can decline them as credential
-      # harvesting and make no call (run 42166675433). A session that ends with
+      # harvesting and make no call. A session that ends with
       # no tool call proved nothing, so it runs again, up to three times; a call
       # made and not denied stops the job at once. The stream can hold what a
       # call returned, so it stays in a file that is never printed and is
@@ -1397,7 +1397,7 @@ jobs:
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
       # instead of reaching a case. The prompt names the run a policy self-test:
       # asked only to read those paths, the model can decline them as credential
-      # harvesting and make no call (run 42166675433). A session that ends with
+      # harvesting and make no call. A session that ends with
       # no tool call proved nothing, so it runs again, up to three times; a call
       # made and not denied stops the job at once. The stream can hold what a
       # call returned, so it stays in a file that is never printed and is
@@ -2182,7 +2182,7 @@ jobs:
       # "Read & Edit"), so a Glob or Grep the policy does not stop fails here
       # instead of reaching a case. The prompt names the run a policy self-test:
       # asked only to read those paths, the model can decline them as credential
-      # harvesting and make no call (run 42166675433). A session that ends with
+      # harvesting and make no call. A session that ends with
       # no tool call proved nothing, so it runs again, up to three times; a call
       # made and not denied stops the job at once. The stream can hold what a
       # call returned, so it stays in a file that is never printed and is