From e97f69c6251803b679ba9829a177ca1eaf9620dc Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Thu, 5 Nov 2026 21:06:46 -0500
Subject: [PATCH 1/2] fix(plugin-eval): read the eval probe's stream line by
 line and name its facts on a failed denial check

Since the eval legs moved to `--output-format stream-json --verbose` (#13626),
the probe's denial check slurped the whole stream with `jq -s`, which fails
on any stdout line that is not JSON. A failed parse reads as "the managed
policy did not record a denied Read", and nothing the run logs can tell that
apart from a policy that let the read through, a model that made no call, or
an API error (run 42146761553, both baseline key legs).

- The check now parses each line with `fromjson?`, so a line that is not
  JSON is skipped. The check is unchanged otherwise and still fails closed.
- A failed check now names the probe's closed facts:
  - whether a result line exists, its subtype, terminal_reason, is_error,
    api_error_status and num_turns;
  - the tools called, the denial count and the count of lines that are
    not JSON.
  It names no text the model or a tool wrote.
- Test: every eval leg in both workflows runs the same managed-policy step
  and the same probe, and the probe parses line by line.

Refs: #13586
Refs: #12035

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .../actions/resolve-config/callers.test.mjs   | 35 ++++++++++++
 .github/workflows/eval-browser-dispatch.yml   | 57 +++++++++++++++++--
 .github/workflows/eval-hill-climb-pass.yml    | 57 +++++++++++++++++--
 3 files changed, 137 insertions(+), 12 deletions(-)

diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs
index cd838bee3a4..2b2c5f44d4b 100644
--- a/.github/actions/resolve-config/callers.test.mjs
+++ b/.github/actions/resolve-config/callers.test.mjs
@@ -9325,6 +9325,41 @@ test("handoff-evidence.sh: credit-stop is the run job's Mark a key credit stop s
   assert.deepEqual(readPosted(t, read, listing([], "skipped")).outputs, { "credit-stop": "false" });
 });
 
+// Every eval leg, in both workflows, runs the same policy step and the same
+// probe: the credential is the only difference, so a key leg cannot drift from
+// the subscription leg the probe was proven on. The probe reads its stream
+// line by line, so one line that is not JSON cannot hide the denials, and a
+// failed check names the probe's closed facts.
+test("every eval leg runs the same managed-policy step and the same probe", () => {
+  const legs = [
+    ["eval-hill-climb-pass.yml", ["pass", "pass-subscription", "pass-fallback"]],
+    ["eval-browser-dispatch.yml", ["eval", "eval-subscription", "eval-fallback"]],
+  ].flatMap(([file, ids]) => ids.map((id) => [file, id, workflow(file).jobs[id].steps]));
+  const step = (steps, id) => steps.find((s) => s.id === id);
+  const [, , first] = legs[0];
+  const policy = (steps) => /\{permissions: \{deny: \[[^]*?\]\}\}/.exec(step(steps, "deny").run)[0];
+  for (const [file, id, steps] of legs) {
+    const own = legs.find(([f]) => f === file)[2];
+    assert.equal(step(steps, "deny").run, step(own, "deny").run, `${file} ${id} managed-policy step`);
+    assert.equal(policy(steps), policy(first), `${file} ${id} deny rules`);
+    // The probe's messages name its workflow's run and case kind, and the
+    // dispatch probe logs its pass; nothing else differs.
+    const probeOf = (s) =>
+      step(s, "probe")
+        .run.replaceAll("no pull request case runs", "no case runs")
+        .replace("the next leg runs the eval", "the next leg runs the pass")
+        .replace(`echo "The managed policy denied the probe's Read, Grep and Glob calls."\n`, "");
+    assert.equal(probeOf(steps), probeOf(first), `${file} ${id} probe`);
+    const names = steps.map((s) => s.name);
+    assert.ok(names.indexOf("Deny the eval's reads of process and runner state") < names.indexOf("Isolate the runner"), `${file} ${id}: the policy is written while sudo works`);
+    assert.ok(names.indexOf("Isolate the runner") < names.indexOf("Probe that the policy denies those reads"), `${file} ${id}`);
+  }
+  const probe = step(first, "probe").run;
+  assert.match(probe, /jq -e -R -s --arg t "\$1" --arg p "\$2" '\[split\("\\n"\)\[\] \| fromjson\? \| objects\] \| last\(/);
+  assert.doesNotMatch(probe, /jq -e -s /, "a slurp fails on any line that is not JSON");
+  assert.match(probe, /so no case runs \(probe: \$\(probe_facts\)\)"; exit 1; \}/);
+});
+
 // --- every model lane runs the three-leg chain (#13586) ---
 
 // A lane is the catalog lanes of one workflow and role that share an id once
diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml
index 7de7e6cddda..7070287019a 100644
--- a/.github/workflows/eval-browser-dispatch.yml
+++ b/.github/workflows/eval-browser-dispatch.yml
@@ -601,9 +601,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
@@ -1406,9 +1421,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
@@ -2165,9 +2195,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml
index 8d2d7d9f2a8..b6fa9d37c7b 100644
--- a/.github/workflows/eval-hill-climb-pass.yml
+++ b/.github/workflows/eval-hill-climb-pass.yml
@@ -354,9 +354,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
@@ -747,9 +762,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"
@@ -1095,9 +1125,24 @@ jobs:
             echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
             exit 0
           fi
+          probe_facts() {
+            jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
+              [split("\n")[] | select(. != "")] as $lines
+              | [$lines[] | fromjson? | objects] as $m
+              | ($m | map(select(.type == "result")) | last) as $r
+              | [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
+                  | if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
+              | "result=\(if $r == null then "none" else "present" end)"
+                + " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+                + " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+                + " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+                + " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+                + " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+                + " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
+          }
           denied() {
-            jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
-              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
+            jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
+              { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
           }
           denied Read /proc/self/environ
           denied Read "$RUNNER_ROOT/.runner"

From 80b30ff9d47f8605ae44dc2b884581115542597e Mon Sep 17 00:00:00 2001
From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com>
Date: Thu, 5 Nov 2026 23:57:06 -0500
Subject: [PATCH 2/2] test(plugin-eval): fail the probe-order check on a
 missing step name

indexOf returns -1 for a renamed step, which compares smaller than any
real index, so the deny-before-isolate check passed without checking.
Each step name must now be found before the order is compared.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/actions/resolve-config/callers.test.mjs | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs
index 787f812e9f6..70363fe5ca5 100644
--- a/.github/actions/resolve-config/callers.test.mjs
+++ b/.github/actions/resolve-config/callers.test.mjs
@@ -9417,9 +9417,13 @@ test("every eval leg runs the same managed-policy step and the same probe", () =
         .replace("the next leg runs the eval", "the next leg runs the pass")
         .replace(`echo "The managed policy denied the probe's Read, Grep and Glob calls."\n`, "");
     assert.equal(probeOf(steps), probeOf(first), `${file} ${id} probe`);
-    const names = steps.map((s) => s.name);
-    assert.ok(names.indexOf("Deny the eval's reads of process and runner state") < names.indexOf("Isolate the runner"), `${file} ${id}: the policy is written while sudo works`);
-    assert.ok(names.indexOf("Isolate the runner") < names.indexOf("Probe that the policy denies those reads"), `${file} ${id}`);
+    const [denyIdx, isolateIdx, probeIdx] = ["Deny the eval's reads of process and runner state", "Isolate the runner", "Probe that the policy denies those reads"].map((n) => {
+      const i = steps.findIndex((s) => s.name === n);
+      assert.ok(i >= 0, `${file} ${id}: no step named ${n}`);
+      return i;
+    });
+    assert.ok(denyIdx < isolateIdx, `${file} ${id}: the policy is written while sudo works`);
+    assert.ok(isolateIdx < probeIdx, `${file} ${id}`);
   }
   const probe = step(first, "probe").run;
   assert.match(probe, /jq -e -R -s --arg t "\$1" --arg p "\$2" '\[split\("\\n"\)\[\] \| fromjson\? \| objects\] \| last\(/);