Skip to content

fix(plugin-eval): read the eval probe's stream line by line and name its facts on a failed denial check #13640

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kyle-sexton merged 4 commits into from
Nov 6, 2026
Merged

fix(plugin-eval): read the eval probe's stream line by line and name its facts on a failed denial check #13640

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e97f69c
fix(plugin-eval): read the eval probe's stream line by line and name …
kyle-sexton Nov 6, 2026
228e9ce
Merge remote-tracking branch 'origin/main' into fix/eval-probe-denial…
kyle-sexton Nov 6, 2026
9fb5565
Merge branch 'main' into fix/eval-probe-denial-read
kyle-sexton Nov 6, 2026
80b30ff
test(plugin-eval): fail the probe-order check on a missing step name
kyle-sexton Nov 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions .github/actions/resolve-config/callers.test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9392,6 +9392,45 @@ test("handoff-evidence.sh: credit-stop is the run job's Mark a key credit stop s
assert.deepEqual(readPosted(t, read, listing([], "skipped")).outputs, { "credit-stop": "false" });
});

// Every eval leg, in both workflows, runs the same policy step and the same
// probe: the credential is the only difference, so a key leg cannot drift from
// the subscription leg the probe was proven on. The probe reads its stream
// line by line, so one line that is not JSON cannot hide the denials, and a
// failed check names the probe's closed facts.
test("every eval leg runs the same managed-policy step and the same probe", () => {
const legs = [
["eval-hill-climb-pass.yml", ["pass", "pass-subscription", "pass-fallback"]],
["eval-browser-dispatch.yml", ["eval", "eval-subscription", "eval-fallback"]],
].flatMap(([file, ids]) => ids.map((id) => [file, id, workflow(file).jobs[id].steps]));
const step = (steps, id) => steps.find((s) => s.id === id);
const [, , first] = legs[0];
const policy = (steps) => /\{permissions: \{deny: \[[^]*?\]\}\}/.exec(step(steps, "deny").run)[0];
for (const [file, id, steps] of legs) {
const own = legs.find(([f]) => f === file)[2];
assert.equal(step(steps, "deny").run, step(own, "deny").run, `${file} ${id} managed-policy step`);
assert.equal(policy(steps), policy(first), `${file} ${id} deny rules`);
// The probe's messages name its workflow's run and case kind, and the
// dispatch probe logs its pass; nothing else differs.
const probeOf = (s) =>
step(s, "probe")
.run.replaceAll("no pull request case runs", "no case runs")
.replace("the next leg runs the eval", "the next leg runs the pass")
.replace(`echo "The managed policy denied the probe's Read, Grep and Glob calls."\n`, "");
assert.equal(probeOf(steps), probeOf(first), `${file} ${id} probe`);
const [denyIdx, isolateIdx, probeIdx] = ["Deny the eval's reads of process and runner state", "Isolate the runner", "Probe that the policy denies those reads"].map((n) => {
const i = steps.findIndex((s) => s.name === n);
assert.ok(i >= 0, `${file} ${id}: no step named ${n}`);
return i;
});
assert.ok(denyIdx < isolateIdx, `${file} ${id}: the policy is written while sudo works`);
assert.ok(isolateIdx < probeIdx, `${file} ${id}`);
}
const probe = step(first, "probe").run;
assert.match(probe, /jq -e -R -s --arg t "\$1" --arg p "\$2" '\[split\("\\n"\)\[\] \| fromjson\? \| objects\] \| last\(/);
assert.doesNotMatch(probe, /jq -e -s /, "a slurp fails on any line that is not JSON");
assert.match(probe, /so no case runs \(probe: \$\(probe_facts\)\)"; exit 1; \}/);
});

// --- mutated code never runs beside a credential (#13586) ---

// Owner constraint on #13586: no job that runs mutants holds a Claude
Expand Down
57 changes: 51 additions & 6 deletions .github/workflows/eval-browser-dispatch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -601,9 +601,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down Expand Up @@ -1406,9 +1421,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down Expand Up @@ -2165,9 +2195,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down
57 changes: 51 additions & 6 deletions .github/workflows/eval-hill-climb-pass.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,9 +354,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down Expand Up @@ -747,9 +762,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down Expand Up @@ -1095,9 +1125,24 @@ jobs:
echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass"
exit 0
fi
probe_facts() {
jq -r -R -s 'def token: tostring | if test("^[a-z_]{1,40}$") then . else "other" end;
[split("\n")[] | select(. != "")] as $lines
| [$lines[] | fromjson? | objects] as $m
| ($m | map(select(.type == "result")) | last) as $r
| [$m[] | select(.type == "assistant") | .message.content[]? | objects | select(.type == "tool_use") | .name | tostring
| if IN("Read", "Grep", "Glob") then . else "other" end] as $calls
| "result=\(if $r == null then "none" else "present" end)"
+ " subtype=\($r.subtype // "none" | token) terminal_reason=\($r.terminal_reason // "none" | token)"
+ " is_error=\(if $r.is_error == null then "none" else ($r.is_error | tostring) end | if IN("true", "false", "none") then . else "other" end)"
+ " api_error_status=\($r.api_error_status // "none" | tostring | if test("^([0-9]{3}|none)$") then . else "other" end)"
+ " num_turns=\($r.num_turns // "none" | tostring | if test("^([0-9]{1,4}|none)$") then . else "other" end)"
+ " tool_calls=\(if ($calls | length) == 0 then "none" else ($calls | join(",")) end)"
+ " denials=\($r.permission_denials // [] | length) unparsable_lines=\(($lines | length) - ($m | length))"' "$out" 2>/dev/null || echo "unreadable"
}
denied() {
jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; }
jq -e -R -s --arg t "$1" --arg p "$2" '[split("\n")[] | fromjson? | objects] | last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 ||
{ echo "::error::the managed policy did not record a denied $1 of $2, so no case runs (probe: $(probe_facts))"; exit 1; }
}
denied Read /proc/self/environ
denied Read "$RUNNER_ROOT/.runner"
Expand Down
Loading