From c98fe791c1eb0ec08f19136789999e4e02f03e39 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 17:52:00 -0500 Subject: [PATCH 1/4] ci(lanes): run the maintenance eval and propose lanes on the three-leg chain The maintenance role's lanes now run its API key, then its subscription token, then FALLBACK_CLAUDE_CODE_OAUTH_TOKEN, one credential per job or step, with the check-key-credit gate and a lane-outcome record per leg. - maintenance-measure-skill-edit.yml propose: a key leg before the subscription leg, a propose-key-credit gate, and the key's credit-stop marker. - eval-hill-climb-pass.yml and eval-browser-dispatch.yml: pass and eval split into key, subscription and fallback jobs behind a credit gate. eval-handoff.mjs reads the probe's stream and every errored run's kept trace with run-claude-cli's own classifier (lane.cjs judgeLeg and classify.cjs isCreditStop at b8bdd7f) and hands off on the same stops. - lane-funding.json, the retention table and the guard tests follow; the maintenance LEG_EXEMPTIONS entries are gone. Refs: #13586 Co-Authored-By: Claude Opus 5.5 --- .../actions/resolve-config/callers.test.mjs | 142 +- .github/artifact-retention.json | 24 + .github/lane-funding.json | 16 +- .github/scripts/plugin-eval/eval-handoff.mjs | 155 ++ .../scripts/plugin-eval/eval-handoff.test.mjs | 131 ++ .../plugin-eval/hill-climb-workflow.test.mjs | 34 +- .github/scripts/plugin-eval/workflow.test.mjs | 13 +- .github/workflows/eval-browser-dispatch.yml | 1768 ++++++++++++++++- .github/workflows/eval-hill-climb-pass.yml | 943 ++++++++- .../maintenance-measure-skill-edit.yml | 222 ++- 10 files changed, 3354 insertions(+), 94 deletions(-) create mode 100644 .github/scripts/plugin-eval/eval-handoff.mjs create mode 100644 .github/scripts/plugin-eval/eval-handoff.test.mjs diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 2357e5c1def..4bd492b4278 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -2145,14 +2145,19 @@ test("every job that reads a Claude secret is a catalog lane, and every catalog // which only a call of a ci-workflows reusable with a fallback job may pass // (intake-triage.yml and pr-review-cli.yml from v0.65.12, ci-workflows#890), // or a job that runs the fallback leg itself, passing it to run-claude-cli as -// fallback-token or to the read runner as its third call's -// fallback-claude-code-oauth-token. Any job may ask only whether it is set. +// fallback-token, to the read runner as its third call's +// fallback-claude-code-oauth-token, or to a `claude plugin eval` leg's claude +// steps as CLAUDE_CODE_OAUTH_TOKEN; or a call of the eval pass workflow, which +// runs that leg. Any job may ask only whether it is set. const FALLBACK_SECRET = "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"; const FALLBACK_REUSABLE = /^melodic-software\/ci-workflows\/\.github\/workflows\/(?:intake-triage|pr-review-cli)\.yml@main$/; +const EVAL_PASS_REUSABLE = "./.github/workflows/eval-hill-climb-pass.yml"; const FALLBACK_VALUE = `\${{ secrets.${FALLBACK_SECRET} }}`; const readsFallback = (job) => (FALLBACK_REUSABLE.test(String(job.uses ?? "")) && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || + (job.uses === EVAL_PASS_REUSABLE && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || (job.steps ?? []).some((step) => isLauncherStep(step, "run-claude-cli") && step.with?.["fallback-token"] === FALLBACK_VALUE) || + (job.steps ?? []).some((step) => isLauncherStep(step, "claude-cli") && step.env?.CLAUDE_CODE_OAUTH_TOKEN === FALLBACK_VALUE) || (job.uses === READ_RUNNER && job.with?.fallback === "fallback" && job.secrets?.["fallback-claude-code-oauth-token"] === FALLBACK_VALUE) || (claudeReads(job).includes(FALLBACK_SECRET) && !claudeReads(withoutPresenceChecks(job)).includes(FALLBACK_SECRET)); test("every catalog lane reads only its role's secrets, and every role: it passes is its role", () => { @@ -2178,11 +2183,23 @@ test("every catalog lane reads only its role's secrets, and every role: it passe } }); -test("a job reads a Claude API key only when its catalog lane runs on run-claude-cli", () => { +// The maintenance role's `claude plugin eval` lanes take its key too (owner +// decision Q2 on #13586): plugin eval runs on the credential in the +// environment, ANTHROPIC_API_KEY for a Console key +// (code.claude.com/docs/en/plugin-evals, "Run evals in CI"). +test("a job reads a Claude API key only when its catalog lane runs on run-claude-cli or is a plugin eval leg", () => { const lanes = new Map(catalogLanes().map(({ lane, key }) => [key, lane])); const keyed = [...claudeSecretJobs()].filter(([, { names }]) => names.some(isApiKey)); assert.ok(keyed.length > 0); - for (const [key] of keyed) assert.equal(lanes.get(key)?.launcher, "run-claude-cli", key); + for (const [key, { job }] of keyed) { + const launcher = lanes.get(key)?.launcher; + if (launcher === "claude-cli") { + const evalRun = (job.steps ?? []).some((step) => isLauncherStep(step, "claude-cli") && /^\s*claude plugin eval /m.test(step.run)); + assert.ok(evalRun || job.uses === EVAL_PASS_REUSABLE, `${key} reads a key but runs no claude plugin eval`); + } else { + assert.equal(launcher, "run-claude-cli", key); + } + } }); test("a Claude credential reaches only the launcher step of its catalog lane", () => { @@ -9032,6 +9049,116 @@ test("maintenance-tidy-code-upkeep.yml: the sweep runs key, subscription and fal for (const id of ["model-subscription", "model-fallback"]) assert.doesNotMatch(JSON.stringify(jobs[id]), /key-credit-stop/, id); }); +// The maintenance role's `claude plugin eval` lanes run key, subscription and +// fallback legs, one credential per job, each the key leg's steps but for the +// credential, the leg its handoff judge names, the key's credit-stop marker +// and the leg record's credential and fallback reason. A key reaches the CLI +// as ANTHROPIC_API_KEY only inside the claude step, LANE_API_KEY unset first, +// so CLAUDE_CODE_SUBPROCESS_ENV_SCRUB strips it from Bash subprocesses. +const EVAL_LEG_CREDENTIALS = { + key: ["LANE_API_KEY", "${{ secrets.MAINTENANCE_CLAUDE_API_KEY }}"], + subscription: ["CLAUDE_CODE_OAUTH_TOKEN", "${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }}"], + fallback: ["CLAUDE_CODE_OAUTH_TOKEN", "${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }}"], +}; +const CLASSIFIER_PIN = "b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6"; +for (const [file, gateId, [keyId, subId, fbId], marker] of [ + ["eval-hill-climb-pass.yml", "gate", ["pass", "pass-subscription", "pass-fallback"], "key-credit-stop-maintenance-measure-skill-edit.yml-pass"], + ["eval-browser-dispatch.yml", "eval-key-credit", ["eval", "eval-subscription", "eval-fallback"], "key-credit-stop-eval-browser-dispatch.yml-eval"], +]) { + test(`${file}: the eval runs key, subscription and fallback legs, each with one credential`, () => { + const { jobs } = workflow(file); + const legs = { key: keyId, subscription: subId, fallback: fbId }; + for (const [leg, id] of Object.entries(legs)) { + const steps = jobs[id].steps; + const launchers = steps.filter((s) => isLauncherStep(s, "claude-cli")); + assert.ok(launchers.some((s) => /^\s*claude plugin eval /m.test(s.run)), `${id} runs the eval`); + const [name, value] = EVAL_LEG_CREDENTIALS[leg]; + for (const step of launchers) { + assert.equal(step.env[name], value, `${id} ${step.name}`); + assert.deepEqual(stepSecrets(step), [value.match(/secrets\.(\w+)/)[1]], `${id} ${step.name} holds one credential`); + const firstClaude = step.run.search(/^\s*claude\s/m); + assert.ok(step.run.indexOf("unset LANE_API_KEY") < firstClaude, `${id} ${step.name} unsets LANE_API_KEY before claude runs`); + assert.match(step.env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB, /^(?:"?1"?|\$\{\{ .+ \}\})$/, `${id} ${step.name} scrubs Bash subprocesses`); + } + assert.equal(steps.find((s) => s.id === "judge").env.LEG, leg, id); + const checkout = steps.find((s) => s.with?.repository === "melodic-software/ci-workflows"); + assert.equal(checkout.with.ref, CLASSIFIER_PIN, `${id} reads the classifier at the run-claude-cli pin`); + assert.equal(checkout.with["persist-credentials"], false); + assert.ok(!jobs[id].permissions || !Object.values(jobs[id].permissions).includes("write"), `${id} holds no write grant`); + const record = steps.find((s) => s.name === "Record the leg"); + assert.equal(record.with.credential, value.match(/secrets\.(\w+)/)[1], id); + assert.equal(record.with["execution-file"], "${{ steps.judge.outputs.leg-result }}", id); + } + // The run-claude-cli pin the classifier checkout follows is the one this repository runs. + assert.match(read("../../workflows/maintenance-measure-skill-edit.yml"), new RegExp(`run-claude-cli@${CLASSIFIER_PIN} `)); + + const shape = (id) => + jobs[id].steps + .filter((s) => !["credential", "credit-marker", "credit-marker-scan", "credit-marker-upload"].includes(s.id)) + .map((s) => { + const copy = structuredClone(s); + for (const k of ["LANE_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", "LEG"]) delete copy.env?.[k]; + if (copy.name === "Record the leg") for (const k of ["credential", "fallback-reason"]) delete copy.with[k]; + return copy; + }); + assert.deepEqual(shape(subId), shape(keyId)); + assert.deepEqual(shape(fbId), shape(keyId)); + assert.deepEqual(Object.keys(jobs[subId].outputs), Object.keys(jobs[keyId].outputs)); + + const ctx = (over) => ({ + [`needs.${gateId}.result`]: "success", + [`needs.${gateId}.outputs.skip-key`]: "", + "github.event_name": "workflow_dispatch", + "github.ref": "refs/heads/main", + "github.event.repository.default_branch": "main", + "github.event.label.name": "", + "github.event.pull_request.head.repo.full_name": "", + "github.repository": "melodic-software/claude-code-plugins", + "github.event.pull_request.draft": false, + ...over, + }); + const keyIf = jobs[keyId].if.replaceAll("format('refs/heads/{0}', github.event.repository.default_branch)", "'refs/heads/main'"); + assert.equal(evaluate(keyIf, ctx({})), true); + assert.equal(evaluate(keyIf, ctx({ [`needs.${gateId}.result`]: "failure" })), false); + const sub = (over) => + evaluate(jobs[subId].if, ctx({ [`needs.${keyId}.outputs.handoff`]: "", [`needs.${keyId}.outputs.credential-present`]: "true", ...over })); + assert.equal(sub({ [`needs.${keyId}.outputs.handoff`]: "true" }), true, "a handed-off key"); + assert.equal(sub({ [`needs.${keyId}.outputs.credential-present`]: "false" }), true, "an empty key"); + assert.equal(sub({ [`needs.${gateId}.outputs.skip-key`]: "true" }), true, "a credit-stopped key"); + assert.equal(sub({ [`needs.${keyId}.outputs.handoff`]: "false" }), false, "a key leg that finished"); + assert.equal(sub({}), false, "a key leg that failed before its judge"); + const fb = (handoff, present, ready) => + evaluate(jobs[fbId].if, { + [`needs.${subId}.outputs.handoff`]: handoff, + [`needs.${subId}.outputs.credential-present`]: present, + [`needs.${subId}.outputs.fallback-ready`]: ready, + }); + assert.equal(fb("true", "true", "true"), true); + assert.equal(fb("", "false", "true"), true, "an empty subscription token hands to the fallback leg"); + assert.equal(fb("", "false", "false"), false, "the fallback leg starts only when its token is set"); + assert.equal(fb("true", "true", "false"), false); + assert.equal(fb("false", "true", "true"), false); + assert.equal( + jobs[subId].steps.find((s) => s.name === "Record the leg").with["fallback-reason"], + `\${{ needs.${keyId}.outputs.handoff-reason || needs.${gateId}.outputs.skip-key == 'true' && 'credit-stop' || needs.${keyId}.outputs.credential-present == 'false' && 'no-key' || '' }}`, + ); + assert.equal(jobs[fbId].steps.find((s) => s.name === "Record the leg").with["fallback-reason"], `\${{ needs.${subId}.outputs.handoff-reason || 'no-key' }}`); + + // The gate reads markers with actions: read and nothing else; only the key leg uploads one. + assert.deepEqual(jobs[gateId].permissions, { actions: "read" }); + assert.equal(claudeReads(withoutPresenceChecks(jobs[gateId])).length, 0, "the gate reads no Claude credential"); + const credit = jobs[gateId].steps.find((s) => s.id === "credit"); + assert.equal(credit.with["key-job"], keyId); + assert.equal(credit.with["probe-minutes"], "0"); + const upload = jobs[keyId].steps.find((s) => s.id === "credit-marker-upload"); + assert.equal(upload.with.name, marker); + assert.equal(upload["continue-on-error"], true); + assert.match(jobs[keyId].steps.find((s) => s.id === "credit-marker").if, new RegExp(`'${marker.replaceAll(".", "\\.")}' && steps\\.judge\\.outputs\\.handoff-reason == 'credit-stop'`)); + for (const id of [subId, fbId]) assert.doesNotMatch(JSON.stringify(jobs[id]), /key-credit-stop/, id); + assert.equal(JSON.parse(read("../../artifact-retention.json")).families[marker]?.["retention-days"], 1, `${marker} has a retention row`); + }); +} + // --- every model lane runs the three-leg chain (#13586) --- // A lane is the catalog lanes of one workflow and role that share an id once @@ -9063,18 +9190,11 @@ function missingLegs(catalog, launched) { } return missing; } -const EVAL_REASON = "the maintenance role holds no API key (keyed false, no MAINTENANCE_CLAUDE_API_KEY), and claude plugin eval runs outside run-claude-cli, so it has no handoff to a fallback leg (#13586)"; // Lanes that cannot take every leg yet (#13586): each names the exact legs it // waives and why. The test fails when an exempt lane lacks a leg its entry // does not name, or runs a leg its entry waives, so an entry never covers // more than it says and goes once its lane gains the leg. const LEG_EXEMPTIONS = { - "eval-browser-dispatch.yml maintenance eval": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "eval-hill-climb-pass.yml maintenance pass": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-a": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-a2": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-b": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance propose": { legs: ["key"], reason: "the maintenance role holds no API key (keyed false, no MAINTENANCE_CLAUDE_API_KEY) (#13586)" }, "post-merge-test-mutation.yml automations audit": { legs: ["key", "fallback"], reason: "it runs on claude-code-action, which takes no API key here (only run-claude-cli does) and has no handoff to a fallback leg; both wait on the audit's move to run-claude-cli (#13586)", diff --git a/.github/artifact-retention.json b/.github/artifact-retention.json index 0459d17b96b..4c93cccef9d 100644 --- a/.github/artifact-retention.json +++ b/.github/artifact-retention.json @@ -75,6 +75,30 @@ "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", "scan": "upload" }, + "key-credit-stop-eval-browser-dispatch.yml-eval": { + "retention-days": 1, + "readers": [ + ".github/workflows/eval-browser-dispatch.yml eval-key-credit's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention)" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, + "key-credit-stop-maintenance-measure-skill-edit.yml-pass": { + "retention-days": 1, + "readers": [ + ".github/workflows/eval-hill-climb-pass.yml gate's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention); the name carries maintenance-measure-skill-edit.yml because a called workflow's github context is its caller's" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, + "key-credit-stop-maintenance-measure-skill-edit.yml-propose": { + "retention-days": 1, + "readers": [ + ".github/workflows/maintenance-measure-skill-edit.yml propose-key-credit's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention)" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, "key-credit-stop-pr-verify-capture.yml-capture": { "retention-days": 1, "readers": [ diff --git a/.github/lane-funding.json b/.github/lane-funding.json index f4e7f76eda6..479047ff177 100644 --- a/.github/lane-funding.json +++ b/.github/lane-funding.json @@ -127,12 +127,16 @@ "keyed": true, "category": "maintenance", "lanes": [ - { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-a", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-a2", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "propose", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "run-claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-b", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] } + { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY"] }, + { "id": "eval-subscription", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "eval-fallback", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-a", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-a2", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "propose", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "run-claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-b", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY"] }, + { "id": "pass-subscription", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-fallback", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] } ] } ] diff --git a/.github/scripts/plugin-eval/eval-handoff.mjs b/.github/scripts/plugin-eval/eval-handoff.mjs new file mode 100644 index 00000000000..4187905dd96 --- /dev/null +++ b/.github/scripts/plugin-eval/eval-handoff.mjs @@ -0,0 +1,155 @@ +#!/usr/bin/env node +// eval-handoff.mjs: whether one `claude plugin eval` leg hands off to the next +// credential, judged by run-claude-cli's own classifier. +// +// A leg is the policy probe (`claude -p --output-format stream-json`) plus the +// eval it guards, both on one credential. Each kept run trace is the session +// as stream-json, one message per line (code.claude.com/docs/en/plugin-evals, +// "What a grader can look at", the `trace` row), the same stream run-claude-cli +// writes as its execution file. So every errored run's trace and the probe's +// stream go through lane.cjs judgeLeg and classify.cjs isCreditStop from the +// ci-workflows checkout at the pin the lanes run (CI_WORKFLOWS), and the leg +// hands off on the rule lane.cjs handoffReason applies, restated below because +// lane.cjs does not export it: +// key leg a credit stop, or class auth with API status 401 or 402 +// subscription leg with FALLBACK_READY true: a credit stop, a 401 or 402, +// or a limit stop (a rejected rate_limit_event with its reset) +// fallback leg never +// A run whose `error` is null ended normally and is not judged; the suite's +// exit code is not read, since a case below --threshold also exits 1. +// +// Env: LEG (key, subscription or fallback), FALLBACK_READY, CI_WORKFLOWS, +// RESULTS (the --json results, one path per line; a missing one is skipped), +// PROBE and PROBE_EXIT (the probe's stream and exit code; PROBE may be empty), +// LEG_RESULT (where to write the leg's execution file for report-lane-outcome: +// one result message carrying the suites' summed durationSeconds and costUsd, +// nothing a run wrote). +// Prints handoff, handoff-reason, lane-class and outcome as key=value lines. +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const LEGS = { key: "api-key", subscription: "subscription", fallback: "fallback" }; +const REFUSED_STATUSES = [401, 402]; +// The classes a run's trace can show that say the credential stopped, not the case. +const STOP_CLASSES = ["credit-stop", "auth", "rate-limit", "overloaded"]; + +export function handoffReason(leg, failureClass, status, { fallbackReady = false, limitStop = false, creditStop = false } = {}) { + const refused = failureClass === "auth" && REFUSED_STATUSES.includes(status); + if (leg === "api-key") { + if (failureClass === "credit-stop") return "credit-stop"; + return refused ? "auth" : ""; + } + if (leg !== "subscription" || !fallbackReady || failureClass === "") return ""; + if (creditStop) return "credit-stop"; + if (refused) return "auth"; + return failureClass === "rate-limit" && limitStop ? "limit-stop" : ""; +} + +// Every errored run's trace path, resolved as run-validity.py resolves it. +export function erroredTraces(result, base) { + const traces = []; + for (const kase of Array.isArray(result?.cases) ? result.cases : []) { + for (const runs of Object.values(kase?.arms ?? {})) { + for (const run of Array.isArray(runs) ? runs : []) { + if (run?.error === null || run?.error === undefined) continue; + const path = run.tracePath; + if (typeof path === "string" && path !== "") traces.push(isAbsolute(path) ? path : join(base, path)); + } + } + } + return traces; +} + +// One stream's verdict: its class and why it hands off ("" when it does not). +export function judgeStream(text, exitCode, { leg, fallbackReady }, { judgeLeg, linesToMessages, isCreditStop }) { + const dir = mkdtempSync(join(tmpdir(), "eval-handoff-")); + try { + const executionFile = join(dir, "execution.json"); + writeFileSync(executionFile, JSON.stringify(linesToMessages(text))); + const judged = judgeLeg({ leg: { leg }, executionFile, exitCode, timedOut: false, maxTurns: "" }); + const status = judged.messages.at(-1)?.api_error_status; + const reason = handoffReason(leg, judged.failureClass, Number.isInteger(status) ? status : null, { + fallbackReady, + limitStop: judged.failureClass === "rate-limit" && judged.resets !== null, + creditStop: leg === "subscription" && isCreditStop(judged.messages, { exitCode }), + }); + return { failureClass: judged.failureClass, reason }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// The leg's verdict over the probe and every errored run: the first stream +// that hands off decides; otherwise the first stop class any stream shows. +export function judgeLegStreams(streams, options, launcher) { + let laneClass = ""; + for (const { text, exitCode } of streams) { + const { failureClass, reason } = judgeStream(text, exitCode, options, launcher); + if (reason !== "") return { handoff: true, reason, laneClass: failureClass }; + if (laneClass === "" && STOP_CLASSES.includes(failureClass)) laneClass = failureClass; + } + return { handoff: false, reason: "", laneClass }; +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function main(env) { + const leg = LEGS[env.LEG]; + if (!leg) throw new Error(`LEG must be key, subscription or fallback, not ${JSON.stringify(env.LEG)}`); + const root = env.CI_WORKFLOWS ?? ""; + const require = createRequire(join(root, ".github/actions/run-claude-cli/lane.cjs")); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + const streams = []; + if (env.PROBE && existsSync(env.PROBE)) { + const exitCode = Number.parseInt(env.PROBE_EXIT ?? "", 10); + streams.push({ text: readFileSync(env.PROBE, "utf8"), exitCode: Number.isInteger(exitCode) ? exitCode : 1 }); + } + let seconds = null; + let cost = null; + const add = (sum, value) => (typeof value === "number" && value >= 0 ? (sum ?? 0) + value : sum); + for (const file of (env.RESULTS ?? "").split("\n").filter(Boolean)) { + const result = readJson(file); + seconds = add(seconds, result?.durationSeconds); + cost = add(cost, result?.costUsd); + for (const trace of erroredTraces(result, dirname(file))) { + if (existsSync(trace)) streams.push({ text: readFileSync(trace, "utf8"), exitCode: 1 }); + } + } + const verdict = judgeLegStreams(streams, { leg, fallbackReady: env.FALLBACK_READY === "true" }, { judgeLeg, linesToMessages, isCreditStop }); + if (env.LEG_RESULT) { + const message = { + type: "result", + subtype: verdict.laneClass === "" ? "success" : "error_during_execution", + is_error: verdict.laneClass !== "", + ...(seconds === null ? {} : { duration_ms: Math.round(seconds * 1000) }), + ...(cost === null ? {} : { total_cost_usd: cost }), + }; + writeFileSync(env.LEG_RESULT, `${JSON.stringify([message])}\n`); + } + return [ + `handoff=${verdict.handoff}`, + `handoff-reason=${verdict.reason}`, + `lane-class=${verdict.laneClass}`, + `outcome=${verdict.laneClass === "" ? "success" : "failure"}`, + ].join("\n"); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + console.log(main(process.env)); + } catch (error) { + console.error(`eval-handoff: ${error.message}`); + process.exit(2); + } +} diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs new file mode 100644 index 00000000000..82ffd090aa8 --- /dev/null +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -0,0 +1,131 @@ +// eval-handoff.mjs hands an eval leg to the next credential on exactly the +// stops run-claude-cli's action.yml names: a key leg's credit stop or 401/402, +// a subscription leg's credit stop, 401/402 or limit stop when a fallback leg +// follows, and never from the fallback leg. The rule cases take their expected +// values from that description (ci-workflows .github/actions/run-claude-cli/ +// action.yml at b8bdd7f, the `handoff` output). The stream cases run the real +// lane.cjs and classify.cjs when EVAL_HANDOFF_CI_WORKFLOWS names a ci-workflows +// checkout at that pin; their messages are the observed shapes ci-workflows' +// classify.test.cjs records (CLI 2.1.302). +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { erroredTraces, handoffReason, judgeLegStreams } from "./eval-handoff.mjs"; + +test("a key leg hands off on a credit stop or a 401 or 402, never a 403 or a limit stop", () => { + assert.equal(handoffReason("api-key", "credit-stop", 400), "credit-stop"); + assert.equal(handoffReason("api-key", "auth", 401), "auth"); + assert.equal(handoffReason("api-key", "auth", 402), "auth"); + assert.equal(handoffReason("api-key", "auth", 403), "", "a 403 stops the run"); + assert.equal(handoffReason("api-key", "auth", null), "", "auth with no status"); + assert.equal(handoffReason("api-key", "rate-limit", 429, { limitStop: true }), ""); + assert.equal(handoffReason("api-key", "", null), ""); +}); + +test("a subscription leg hands off only when a fallback leg follows", () => { + const ready = { fallbackReady: true }; + assert.equal(handoffReason("subscription", "rate-limit", null, { ...ready, limitStop: true }), "limit-stop"); + assert.equal(handoffReason("subscription", "rate-limit", 429, { ...ready, limitStop: false }), "", "a 429 with no rejected window"); + assert.equal(handoffReason("subscription", "other", 400, { ...ready, creditStop: true }), "credit-stop"); + assert.equal(handoffReason("subscription", "auth", 401, ready), "auth"); + assert.equal(handoffReason("subscription", "auth", 403, ready), ""); + assert.equal(handoffReason("subscription", "rate-limit", null, { fallbackReady: false, limitStop: true }), "", "no fallback leg"); + assert.equal(handoffReason("fallback", "rate-limit", null, { ...ready, limitStop: true }), "", "the fallback leg is the last"); +}); + +test("only runs that ended with an error are judged, their traces resolved against the result file", () => { + const result = { + cases: [ + { arms: { with: [{ error: null, tracePath: "/k/ok.jsonl" }, { error: "API error", tracePath: "run/2.jsonl" }], without: [{ error: "timed out after 300s", tracePath: "/k/3.jsonl" }] } }, + { arms: { with: [{ error: "no trace kept" }] } }, + ], + }; + assert.deepEqual(erroredTraces(result, "/out"), ["/out/run/2.jsonl", "/k/3.jsonl"]); + assert.deepEqual(erroredTraces(null, "/out"), []); +}); + +test("the first stream that hands off decides, else the first stop class is kept", () => { + const verdicts = { a: ["", ""], b: ["rate-limit", ""], c: ["auth", "auth"], d: ["credit-stop", "credit-stop"] }; + const launcher = { + linesToMessages: (text) => [{ text }], + judgeLeg: ({ executionFile }) => ({ failureClass: verdicts[JSON.parse(readFileSync(executionFile, "utf8"))[0].text][0], resets: null, messages: [{}] }), + isCreditStop: () => false, + }; + const key = { leg: "api-key", fallbackReady: false }; + const streams = (...names) => names.map((text) => ({ text, exitCode: 1 })); + assert.deepEqual(judgeLegStreams(streams("a", "b"), key, launcher), { handoff: false, reason: "", laneClass: "rate-limit" }); + assert.deepEqual(judgeLegStreams(streams("a", "d", "b"), key, launcher), { handoff: true, reason: "credit-stop", laneClass: "credit-stop" }); + assert.deepEqual(judgeLegStreams(streams("a"), key, launcher), { handoff: false, reason: "", laneClass: "" }); + assert.deepEqual(judgeLegStreams([], key, launcher), { handoff: false, reason: "", laneClass: "" }); +}); + +const ROOT = process.env.EVAL_HANDOFF_CI_WORKFLOWS ?? ""; +const real = ROOT === "" ? null : (() => { + const require = createRequire(join(ROOT, ".github/actions/run-claude-cli/lane.cjs")); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + return { judgeLeg, linesToMessages, isCreditStop }; +})(); +const skip = real === null ? "set EVAL_HANDOFF_CI_WORKFLOWS to a ci-workflows checkout at the run-claude-cli pin" : false; +const jsonl = (messages) => messages.map((m) => JSON.stringify(m)).join("\n"); + +const SPEND_LIMIT_TEXT = "API Error: 400 You have reached your specified workspace API usage limits. You will regain access on 2026-11-01 at 00:00 UTC."; +const SPEND_LIMIT = [ + { type: "assistant", error: "unknown", is_api_error_message: true, api_error_status: 400, message: { model: "", content: [{ type: "text", text: SPEND_LIMIT_TEXT }], usage: { input_tokens: 0, output_tokens: 0 } } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, total_cost_usd: 0, api_error_status: 400, terminal_reason: "api_error", result: SPEND_LIMIT_TEXT, usage: { input_tokens: 0, output_tokens: 0 } }, +]; +const REJECTED_TEXT = "Invalid API key ยท Fix external API key"; +const rejected = (status) => [ + { type: "assistant", error: "authentication_failed", is_api_error_message: true, api_error_status: status, message: { model: "", content: [{ type: "text", text: REJECTED_TEXT }], usage: { input_tokens: 0, output_tokens: 0 } } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, total_cost_usd: 0, api_error_status: status, terminal_reason: "api_error", result: REJECTED_TEXT, usage: { input_tokens: 0, output_tokens: 0 } }, +]; +// A subscription run held by a rejected five-hour window, its reset in the future. +const LIMITED = [ + { type: "system", subtype: "init" }, + { type: "rate_limit_event", rate_limit_info: { status: "rejected", resetsAt: 4102444800, rateLimitType: "five_hour" } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, api_error_status: 429, result: "limit", usage: { input_tokens: 0, output_tokens: 0 } }, +]; +const CLEAN = [{ type: "system", subtype: "init" }, { type: "result", subtype: "success", is_error: false, num_turns: 3 }]; + +test("the real classifier: a key leg's observed spend-limit stop and 401 hand off, a 403 does not", { skip }, () => { + const key = { leg: "api-key", fallbackReady: false }; + assert.deepEqual(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], key, real), { handoff: true, reason: "credit-stop", laneClass: "credit-stop" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(rejected(401)), exitCode: 1 }], key, real), { handoff: true, reason: "auth", laneClass: "auth" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(rejected(403)), exitCode: 1 }], key, real), { handoff: false, reason: "", laneClass: "auth" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(CLEAN), exitCode: 0 }], key, real), { handoff: false, reason: "", laneClass: "" }); +}); + +test("the real classifier: a subscription leg's limit stop and credit stop hand off to the fallback leg", { skip }, () => { + const sub = { leg: "subscription", fallbackReady: true }; + assert.deepEqual(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], sub, real), { handoff: true, reason: "limit-stop", laneClass: "rate-limit" }); + assert.equal(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], sub, real).reason, "credit-stop"); + assert.equal(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], { leg: "subscription", fallbackReady: false }, real).handoff, false); + assert.equal(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], { leg: "fallback", fallbackReady: true }, real).handoff, false); +}); + +test("the CLI reads the probe and the errored runs, and writes the leg's duration for the lane record", { skip }, async () => { + const { spawnSync } = await import("node:child_process"); + const dir = mkdtempSync(join(tmpdir(), "eval-handoff-cli-")); + mkdirSync(join(dir, "t")); + writeFileSync(join(dir, "probe.jsonl"), jsonl(CLEAN)); + writeFileSync(join(dir, "t/1.jsonl"), jsonl(LIMITED)); + writeFileSync(join(dir, "results.json"), JSON.stringify({ durationSeconds: 7.5, costUsd: 0.25, cases: [{ arms: { with: [{ error: "API error", tracePath: "t/1.jsonl" }] } }] })); + writeFileSync(join(dir, "other.json"), JSON.stringify({ durationSeconds: 2.5, costUsd: 0.5, cases: [{ arms: { with: [{ error: null, tracePath: "t/1.jsonl" }] } }] })); + const results = [join(dir, "other.json"), join(dir, "results.json"), join(dir, "absent.json")].join("\n"); + const run = (leg) => + spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + encoding: "utf8", + env: { LEG: leg, FALLBACK_READY: "true", CI_WORKFLOWS: ROOT, RESULTS: results, PROBE: join(dir, "probe.jsonl"), PROBE_EXIT: "0", LEG_RESULT: join(dir, "leg.json") }, + }); + const sub = run("subscription"); + assert.equal(sub.status, 0, sub.stderr); + assert.equal(sub.stdout, "handoff=true\nhandoff-reason=limit-stop\nlane-class=rate-limit\noutcome=failure\n"); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "leg.json"), "utf8")), [ + { type: "result", subtype: "error_during_execution", is_error: true, duration_ms: 10000, total_cost_usd: 0.75 }, + ]); + assert.equal(run("fallback").stdout, "handoff=false\nhandoff-reason=\nlane-class=rate-limit\noutcome=failure\n"); + assert.equal(run("nope").status, 2); +}); diff --git a/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs b/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs index 923c49ba898..ec3b70ce59d 100644 --- a/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs +++ b/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs @@ -39,8 +39,15 @@ test("only push-branch and publish hold id-token, each for its one lane token; e const writer = file === "maintenance-measure-skill-edit.yml" && Object.hasOwn(WRITERS, name); const permissions = block.match(/\n {4}permissions:\n((?: {6}.+\n)+)/)?.[1] ?? ""; const keys = [...permissions.matchAll(/^ {6}([a-z-]+): (\w+)/gm)].map(([, key, level]) => `${key}: ${level}`); - // The gate also reads this run's re-runner (check-trusted-trigger). - const reads = name === "gate" ? ["contents: read", "actions: read"] : ["contents: read"]; + // The gate also reads this run's re-runner (check-trusted-trigger); a + // key-credit gate reads only the credit-stop markers, and a pass's + // caller grants its pass workflow's gate that read. + const creditGate = name === "propose-key-credit" || (file === "eval-hill-climb-pass.yml" && name === "gate"); + const reads = creditGate + ? ["actions: read"] + : name === "gate" || /^pass-(?:a|a2|b)$/.test(name) + ? ["contents: read", "actions: read"] + : ["contents: read"]; assert.deepEqual(keys, writer ? ["contents: read", "id-token: write"] : reads, `${file} ${name}`); const mints = [...code(block).matchAll(/request-lane-token[^]*?\n {10}activity: ([a-z-]+)\n/g)].map(([, activity]) => activity); assert.deepEqual(mints, writer ? [WRITERS[name]] : [], `${file} ${name} mints`); @@ -125,19 +132,34 @@ test("every ci-workflows action is pinned to one full commit", () => { assert.ok(pins.length >= 2); // report-lane-outcome is pinned where every lane pins it; the rest at one commit. for (const pin of pins) assert.ok(["b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 b8bdd7f 2026-11-04", "6b9c640bf209c0c7e76f1111c5d18c96707348a2 6b9c640 2026-11-02"].includes(pin), pin); + // The classifier each pass leg checks out is run-claude-cli's, at the same commit. + const classifier = [...pass.matchAll(/repository: melodic-software\/ci-workflows\n {10}ref: (\S+) # (.+)/g)].map(([, sha, note]) => `${sha} ${note}`); + assert.deepEqual(classifier, Array(3).fill("b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 b8bdd7f 2026-11-04")); }); -test("the model steps are read-only, on the maintenance token then the fallback token, and frame what they read as data", () => { - assert.equal(main.match(/run-claude-cli@/g).length, 2); +test("the model steps are read-only, on the maintenance key, then its token, then the fallback token, and frame what they read as data", () => { + assert.equal(main.match(/run-claude-cli@/g).length, 3); assert.doesNotMatch(pass, /run-claude-cli@/); const propose = jobs(main).propose; + const key = step(propose, "Propose the edit on the API key"); const leg = step(propose, "Propose the edit"); const fallback = step(propose, "Propose the edit on the fallback token"); + assert.match(key, /api-key: \$\{\{ secrets\.MAINTENANCE_CLAUDE_API_KEY \}\}\n/); + assert.match(key, /if: steps\.prompt\.outcome == 'success' && steps\.key\.outputs\.key == 'true'\n/); assert.match(leg, /subscription-token: \$\{\{ secrets\.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN \}\}\n {10}fallback-ready: \$\{\{ steps\.credential\.outputs\.fallback-ready \}\}\n/); + // The subscription leg runs after the key leg hands off or does not run. + assert.match(leg, /&& \(steps\.run-key\.outcome == 'skipped' \|\| steps\.run-key\.outputs\.handoff == 'true'\)\n/); assert.match(fallback, /fallback-token: \$\{\{ secrets\.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN \}\}\n/); // The fallback leg runs after the subscription leg hands off or finds its token empty, with the fallback token set. - assert.match(fallback, /steps\.credential\.outputs\.fallback-ready == 'true'\n\s+&& \(steps\.run\.outputs\.handoff == 'true' \|\| steps\.credential\.outputs\.present != 'true'\)/); - for (const model of [leg, fallback]) { + assert.match( + fallback, + /steps\.credential\.outputs\.fallback-ready == 'true'\n\s+&& \(steps\.run\.outputs\.handoff == 'true'\n\s+\|\| steps\.credential\.outputs\.present != 'true' && \(steps\.run-key\.outcome == 'skipped' \|\| steps\.run-key\.outputs\.handoff == 'true'\)\)/, + ); + // A credit-stopped key skips the key leg, and only a key credit stop uploads its marker. + assert.match(step(propose, "Check the API key"), /SKIP_KEY: \$\{\{ needs\.propose-key-credit\.outputs\.skip-key \}\}/); + assert.match(step(propose, "Write the key's credit-stop marker"), /'key-credit-stop-maintenance-measure-skill-edit\.yml-propose' && steps\.run-key\.outputs\.handoff-reason == 'credit-stop'/); + assert.match(jobs(main)["propose-key-credit"], /key-job: propose\n/); + for (const model of [key, leg, fallback]) { assert.match(model, /role: maintenance\n/); assert.match(model, /permission-mode: dontAsk\n/); assert.match(model, /tools: \|\n {12}Read\n {12}Grep\n {12}Glob\n {10}allowed-tools:/); diff --git a/.github/scripts/plugin-eval/workflow.test.mjs b/.github/scripts/plugin-eval/workflow.test.mjs index 9c5c704b15d..a4435583a83 100644 --- a/.github/scripts/plugin-eval/workflow.test.mjs +++ b/.github/scripts/plugin-eval/workflow.test.mjs @@ -38,8 +38,14 @@ test("a pull request run proves the policy denies the reads before any case runs for (const call of ["Read /proc/self/environ", 'Read "$RUNNER_ROOT/.runner"', "Grep /proc/self", "Glob /proc/self"]) { assert.ok(probe.includes(`\n denied ${call}\n`), `the probe checks a denied ${call}`); } - assert.match(probe, /if grep -qF "\$CLAUDE_CODE_OAUTH_TOKEN" "\$out"; then\n[^\n]*\n {12}exit 1/); + // The leak check looks for the leg's own credential, whichever leg runs. + assert.match(probe, /credential=\$\{ANTHROPIC_API_KEY:-\$\{CLAUDE_CODE_OAUTH_TOKEN:-\}\}\n {10}\[ -n "\$credential" \] \|\| /); + assert.match(probe, /if grep -qF "\$credential" "\$out"; then\n[^\n]*\n {12}exit 1/); assert.match(probe, /trap 'rm -f "\$out"' EXIT/); + // A probe the credential's own stop ended hands off before the denial checks, never past them. + assert.ok(probe.indexOf("eval-handoff.mjs") < probe.indexOf("denied Read /proc/self/environ")); + assert.ok(probe.indexOf('if grep -qF "$credential"') < probe.indexOf("eval-handoff.mjs"), "the leak check runs before the classifier"); + assert.match(probe, /env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="\$out"/, "the classifier runs without the credential"); assert.doesNotMatch(probe, /cat "\$out"|jq [^\n]*"\$out"[^\n]*\| *tee/); assert.ok(text.indexOf("- name: Install the locked root toolchain") < text.indexOf("- name: Probe that the policy")); assert.ok(text.indexOf("- name: Probe that the policy") < text.indexOf("- name: Run the plugin eval")); @@ -83,13 +89,14 @@ test("a pull request run redacts its summary and uploads no results", () => { assert.match(step("Summarize the results"), /REDACT: \$\{\{ github\.event_name == 'pull_request' && '--redact' \|\| '' \}\}/); assert.match(step("Summarize the results"), /summarize\.mjs "\$results" --plugin "\$name" "\$\{redact\[@\]\}"/); // Only a dispatched run scans and uploads, and the upload needs the scan's clean output. + // A leg that handed off to the next credential uploads nothing. const scan = step("Scan the upload for credentials"); - assert.match(scan, /if: >-\n {10}always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch'\n/); + assert.match(scan, /if: >-\n {10}always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch' && steps\.judge\.outputs\.handoff != 'true'\n/); assert.match(scan, /\n {8}id: scan-upload\n/); assert.match(scan, /\n {10}path: \$\{\{ runner\.temp \}\}\/eval-results\/\n/); assert.match( step("Upload the eval results"), - /if: >-\n {10}\(always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch'\) && steps\.scan-upload\.outputs\.clean == 'true'\n/, + /if: >-\n {10}\(always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch' && steps\.judge\.outputs\.handoff != 'true'\) && steps\.scan-upload\.outputs\.clean == 'true'\n/, ); assert.ok(text.indexOf("- name: Scan the upload for credentials") < text.indexOf("- name: Upload the eval results")); }); diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml index 658355fbdb0..4d97976555f 100644 --- a/.github/workflows/eval-browser-dispatch.yml +++ b/.github/workflows/eval-browser-dispatch.yml @@ -38,8 +38,17 @@ name: eval-browser-dispatch # whose tree holds no symlink or submodule and whose cases load only the # plugin itself or an unchanged directory inside it. Anything else is refused. # -# The credential is the maintenance role's subscription token. When the secret -# is empty the job ends with a notice and runs nothing. +# The eval runs on the maintenance role's credential chain, one credential per +# job: `eval` on MAINTENANCE_CLAUDE_API_KEY, then `eval-subscription` on +# MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN, then `eval-fallback` on +# FALLBACK_CLAUDE_CODE_OAUTH_TOKEN. A leg hands off on the stops run-claude-cli +# hands off on (eval-handoff.mjs reads the probe's stream and every errored +# run's kept trace with run-claude-cli's own classifier): the key leg on a +# credit stop or a 401 or 402, the subscription leg, when the fallback token is +# set, also on a limit stop; an empty credential hands off too. The leg that +# hands off summarizes and uploads nothing. eval-key-credit skips the key after +# a recent credit stop (ci-workflows check-key-credit). With every credential +# empty the last leg ends with a notice and runs nothing. # # Only an env allowlist (PATH, ANTHROPIC_*, CLAUDE_CODE_*, EVAL_*) reaches an # eval run (code.claude.com/docs/en/plugin-evals, "How runs are isolated", as @@ -119,10 +128,13 @@ concurrency: cancel-in-progress: false jobs: - eval: - # A dispatch on any ref but the default branch skips before any step - # runs, so every case it runs has merged through review. A pull request - # runs only on the `eval: run` label, from this repository, and not as a draft. + # The key's credit gate (ci-workflows check-key-credit, gate.cjs: one + # artifact-list call for the marker the key leg uploads after a credit + # stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour unset) + # starts the eval on the subscription, with no refused key attempt first. + # No secret, no model, no checkout; every failure runs the key (skip-key + # empty). Probing is off. + eval-key-credit: if: >- (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) @@ -131,12 +143,64 @@ jobs: && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft) runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: eval + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" + + # The key leg, on the maintenance role's API key alone. A dispatch on any + # ref but the default branch skips before any step runs, so every case it + # runs has merged through review. A pull request runs only on the + # `eval: run` label, from this repository, and not as a draft. A credit + # stop, or a 401 or 402, in the probe or any run hands the eval to + # eval-subscription; an empty key, or one the gate skipped, runs nothing. + eval: + needs: eval-key-credit + if: >- + ${{ !cancelled() && needs.eval-key-credit.result == 'success' && ((github.event_name == 'workflow_dispatch' + && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + || (github.event_name == 'pull_request' + && github.event.label.name == 'eval: run' + && github.event.pull_request.head.repo.full_name == github.repository + && !github.event.pull_request.draft)) }} + runs-on: ubuntu-24.04 timeout-minutes: 60 cache-mode: read permissions: contents: read # check out the repository; nothing in this job writes pull-requests: read # check-trusted-trigger reads the pull request actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} env: API_BUDGET_TOKEN: github-token npm_config_audit: "false" @@ -265,24 +329,28 @@ jobs: exit 1 fi - # Only whether the secret is set leaves this step, never its value. + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. - name: Check the credential id: credential env: - PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.eval-key-credit.outputs.skip-key }} run: | - if [ "$PRESENT" = true ]; then - echo "present=true" >>"$GITHUB_OUTPUT" - else - echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; no eval runs." - echo "present=false" >>"$GITHUB_OUTPUT" + run=$PRESENT + if [ "$SKIP_KEY" = true ]; then + run=false + echo "::notice::the credit gate skipped MAINTENANCE_CLAUDE_API_KEY; eval-subscription runs the eval" + elif [ "$PRESENT" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; eval-subscription runs the eval" fi + { echo "present=$PRESENT"; echo "run=$run"; } >>"$GITHUB_OUTPUT" # The plugin under test on a pull request: the label-time head, with full # history so the next step diffs this same commit against its base. A # checkout runs no code from it. - name: Check out the pull request head - if: steps.credential.outputs.present == 'true' && github.event_name == 'pull_request' + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -295,7 +363,7 @@ jobs: # dispatch. Each line of `plugins` is a plugins/ path. - name: Choose the plugins id: plugins - if: steps.credential.outputs.present == 'true' + if: steps.credential.outputs.run == 'true' env: EVENT: ${{ github.event_name }} TARGET: ${{ inputs.target }} @@ -470,6 +538,22 @@ jobs: echo "/opt/tools/bin" >> "$GITHUB_PATH" test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + # Proof that the managed policy holds before any pull request case runs, # for each of the three tools a case can use to read a file: one Claude # Code run is told to Read /proc/self/environ and the runner's .runner @@ -480,26 +564,44 @@ jobs: # credential anywhere. Claude Code applies Read deny rules to Glob and # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, # "Read & Edit"), so a Glob or Grep the policy does not stop fails here - # instead of reaching a case. The result can hold what a call returned, + # instead of reaching a case. The stream can hold what a call returned, # so it stays in a file that is never printed and is removed after the - # check. + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. - name: Probe that the policy denies those reads + id: probe if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } out=$(mktemp) trap 'rm -f "$out"' EXIT + status=0 claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ - --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format json >"$out" 2>/dev/null || true - if grep -qF "$CLAUDE_CODE_OAUTH_TOKEN" "$out"; then + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi denied() { - jq -e --arg t "$1" --arg p "$2" 'any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } } denied Read /proc/self/environ @@ -567,9 +669,10 @@ jobs: # run's own sandbox makes read-only to Bash, and within the short length # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). - name: Run the plugin eval - if: steps.stage.outputs.go == 'true' + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} # Strips credentials from Bash subprocesses inside each eval run. # The env-scrub input set false gives "0", the opt-out value Claude # Code's own messages name. @@ -583,6 +686,10 @@ jobs: REAL_SERVERS: ${{ inputs.allow-real-servers }} OUT: ${{ runner.temp }}/eval-results run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi export TMPDIR="$HOME/.ev" mkdir -p "$TMPDIR" args=( @@ -636,6 +743,38 @@ jobs: echo "exit $status" >"$out/exit-code" done <<<"$SUITES" + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT # COMPARABLE per case. A suite with no result fails the job too. # A pull request run prints no case name, error or partial reason: the @@ -644,7 +783,7 @@ jobs: # printed to the job log, with workflow commands off, and kept as # summary.md beside its results.json, which a dispatched run uploads. - name: Summarize the results - if: always() && steps.stage.outputs.go == 'true' + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' env: SUITES: ${{ steps.stage.outputs.suites }} OUT: ${{ runner.temp }}/eval-results @@ -713,7 +852,7 @@ jobs: # omitted when nothing matches. Nothing else from the trace is printed. Trace text is # tool output, so workflow commands are off while it prints. - name: Print Bash error excerpts - if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' env: OUT: ${{ runner.temp }}/eval-results run: | @@ -766,14 +905,14 @@ jobs: - name: Scan the upload for credentials id: scan-upload if: >- - always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] with: path: ${{ runner.temp }}/eval-results/ - name: Upload the eval results if: >- - (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch') && steps.scan-upload.outputs.clean == 'true' + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} @@ -781,6 +920,1581 @@ jobs: if-no-files-found: warn retention-days: 3 + # Read by the next runs' credit gate (eval-key-credit): the artifact its + # `marker` output names (check-key-credit gate.cjs markerName), uploaded + # only after the key leg's credit-stop handoff. The name is written out + # so its retention row claims it. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.eval-key-credit.outputs.marker == 'key-credit-stop-eval-browser-dispatch.yml-eval' && steps.judge.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + id: credit-marker-upload + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-eval-browser-dispatch.yml-eval + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_API_KEY + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: "" + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + + # The subscription leg: the key leg's steps on a fresh runner, on the + # maintenance role's subscription token alone, with no credit-stop marker. + # With FALLBACK_CLAUDE_CODE_OAUTH_TOKEN set, a limit stop, a credit stop, or + # a 401 or 402 hands off to eval-fallback, and so does an empty token. With + # both tokens empty the leg ends with a notice and runs nothing. + # callers.test.mjs holds its steps equal to the key leg's but for the + # credential and the leg record's fallback reason. + eval-subscription: + needs: [eval-key-credit, eval] + if: >- + ${{ !cancelled() && needs.eval-key-credit.result == 'success' + && (needs.eval.outputs.handoff == 'true' || needs.eval.outputs.credential-present == 'false' + || needs.eval-key-credit.outputs.skip-key == 'true') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the repository; nothing in this job writes + pull-requests: read # check-trusted-trigger reads the pull request + actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + API_BUDGET_TOKEN: github-token + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + # npm's fetch timeout and retries, from the one file that sets them. + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Resolve main's tip + id: tip + if: github.event_name == 'pull_request' + env: + TIP_BRANCH: ${{ github.event.repository.default_branch }} + GIT_TERMINAL_PROMPT: "0" + run: | + sha=$(timeout 60 git -C "$RUNNER_TEMP" ls-remote "https://github.com/$GITHUB_REPOSITORY.git" "refs/heads/$TIP_BRANCH" | + awk -v ref="refs/heads/$TIP_BRANCH" '$2 == ref { print $1 }') + [[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "::error::the tip of $TIP_BRANCH did not resolve"; exit 1; } + echo "sha=$sha" >>"$GITHUB_OUTPUT" + # The default branch tip; on a dispatch, the dispatched commit, which the + # job if: holds to the default branch. + - name: Check out the default branch + if: github.event_name != 'pull_request' || steps.tip.outputs.sha != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'pull_request' && steps.tip.outputs.sha || github.sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + # Skips the run while Actions or npm installs report a degraded component, + # with a logged reason; fails open (.github/actions/check-upstream-status). + - name: Check upstream status + id: upstream + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + timeout-minutes: 2 + uses: ./.base/.github/actions/check-upstream-status # zizmor: ignore[self-repository] + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + - name: Stop while upstream is degraded + if: steps.upstream.outputs.degraded == 'true' + env: + REASON: ${{ steps.upstream.outputs.reason }} + run: | + echo "::error::upstream-degraded: ${REASON} on its status page stops this run; dispatch it again once it recovers" + exit 1 + + # check-trusted-trigger needs a PR, and a dispatch names none, so the + # dispatcher is matched against the same list directly, by numeric id: + # it must be a listed human, and the lanes App bot is denied by id, as + # in pr-run-activity-write.yml. A re-run is refused: its re-runner's id + # is only in the run's record, which this job's token cannot read, and + # a login can pass to another account. Dispatch the workflow again + # instead. + - name: Check the dispatcher is a trusted human + if: github.event_name == 'workflow_dispatch' + env: + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + ACTOR_ID: ${{ github.actor_id }} + APP_SENDER_ID: ${{ vars.AUTOMATION_LANES_APP_SENDER_ID }} + run: | + if [ "$GITHUB_RUN_ATTEMPT" != 1 ]; then + echo "::error::rerun-refused: a re-run of this workflow is not supported; dispatch it again" + exit 1 + fi + if ! [[ $APP_SENDER_ID =~ ^[1-9][0-9]*$ ]]; then + echo "::error::the AUTOMATION_LANES_APP_SENDER_ID variable must hold the lanes App bot's numeric user id" + exit 1 + fi + if [ "$ACTOR_ID" = "$APP_SENDER_ID" ]; then + echo "::error::bot-actor: the lanes App bot cannot dispatch this lane" + exit 1 + fi + if ! jq -e --argjson id "$ACTOR_ID" ' + .version == 1 + and any(.actors[]; .id == $id and .kind == "human") + ' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the dispatcher is not a listed human" + exit 1 + fi + + - name: Check the pull request trigger + id: gate + if: github.event_name == 'pull_request' + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + + # check-trusted-trigger accepts any listed actor; the label that spends + # the credential must come from a listed human, so a person opts in to + # each run. + - name: Stop on an untrusted pull request trigger + if: github.event_name == 'pull_request' + env: + PROCEED: ${{ steps.gate.outputs.proceed }} + REASON: ${{ steps.gate.outputs.reason }} + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + SENDER_ID: ${{ github.event.sender.id }} + run: | + if [ "$PROCEED" != true ]; then + echo "::error::the trigger is not trusted (reason: ${REASON:-unreadable})" + exit 1 + fi + if ! jq -e --argjson id "$SENDER_ID" 'any(.actors[]; .id == $id and .kind == "human")' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the 'eval: run' label did not come from a listed human" + exit 1 + fi + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + if [ "$FALLBACK_PRESENT" = true ]; then + echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; eval-fallback runs the eval" + else + echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are empty; no eval runs." + fi + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" + + # The plugin under test on a pull request: the label-time head, with full + # history so the next step diffs this same commit against its base. A + # checkout runs no code from it. + - name: Check out the pull request head + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: .head + fetch-depth: 0 + persist-credentials: false + + # The checked-out head's diff from its merge base with the pull + # request's base chooses the plugins; the target input does on a + # dispatch. Each line of `plugins` is a plugins/ path. + - name: Choose the plugins + id: plugins + if: steps.credential.outputs.run == 'true' + env: + EVENT: ${{ github.event_name }} + TARGET: ${{ inputs.target }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ "$EVENT" = workflow_dispatch ]; then + target=${TARGET%/} + if ! [[ $target =~ ^plugins/[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::target must be a plugin directory, plugins/" + exit 1 + fi + selected=$target + else + merge_base=$(git -C .head merge-base "$BASE_SHA" "$HEAD_SHA") + files=$(git -C .head diff --no-renames --name-only "$merge_base" "$HEAD_SHA") + selected=$(printf '%s\n' "$files" | + node .base/.github/scripts/plugin-eval/select-plugins.mjs --repo .head --base "$merge_base" --head "$HEAD_SHA") || { + echo "::error::the pull request changes more of a selected plugin than its suite may run with the credential" + exit 1 + } + fi + if [ -z "$selected" ]; then + echo "::notice::the pull request changes no plugin's skills, agents or evals; no eval runs." + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "list<>"$GITHUB_OUTPUT" + + # validate-cases.py makes no model call. A FAIL is a case the binary + # rejects at load, so the job stops before any spend. On a deterministic + # run each suite is then copied without its judged cases. + - name: Validate and stage the suites + id: stage + if: steps.plugins.outputs.go == 'true' + env: + LIST: ${{ steps.plugins.outputs.list }} + ROOT: ${{ github.event_name == 'pull_request' && '.head' || '.base' }} + DETERMINISTIC: ${{ github.event_name == 'pull_request' || inputs.deterministic-only }} + run: | + staged="" + while IFS= read -r plugin; do + [ -n "$plugin" ] || continue + dir="$ROOT/$plugin" + eval_dir=$(jq -r '.experimental.evals // "evals"' "$dir/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + if [ ! -d "$dir/$eval_dir" ]; then + echo "::notice::$plugin has no eval suite; skipped." + continue + fi + echo "== validate $plugin/$eval_dir" + if ! python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$dir/$eval_dir"; then + echo "::error::$plugin/$eval_dir does not pass validate-cases.py, so no eval runs" + exit 1 + fi + if [ "$DETERMINISTIC" = true ]; then + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$dir" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::notice::$plugin has no case with deterministic graders only; skipped." + continue + fi + eval_dir=evals-deterministic + fi + staged+="$dir"$'\t'"$eval_dir"$'\n' + done <<<"$LIST" + if [ -z "$staged" ]; then + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "suites<>"$GITHUB_OUTPUT" + + # The tools and Chromium go under /opt, outside HOME, which each eval + # run replaces with a temporary one. Only these directories are made + # while sudo works; no third-party code runs before isolation. + - name: Create the tool directories + if: steps.stage.outputs.go == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools /opt/pw + + # A pull request supplies the prompts, and the eval's Read, Glob and Grep + # run without the Bash sandbox, so nothing else keeps them from the + # credential in /proc//environ or the runner's own files. A + # root-owned managed settings file, which applies inside every eval run + # and which nothing in the run overrides (code.claude.com/docs/en/ + # plugin-evals, "How runs are isolated"; code.claude.com/docs/en/ + # settings, "Settings precedence", as of 2026-10-27), denies those + # paths. The runner's install directory is read from the Runner.Worker + # process above this step, not guessed; the job's _temp and _actions + # directories are denied beside it. Written while sudo is still there; + # the job stops if the runner directory is not found or the file cannot + # be written or read back. The probe step below proves the denial. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions." + + # Installs the bubblewrap and socat the Bash sandbox needs, lifts the + # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo + # and docker access for the rest of the job. + - name: Isolate the runner + if: steps.stage.outputs.go == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + # Every binary this job runs comes from the locked root toolchain. Its + # install scripts run here, after sudo and docker access are gone. + - name: Install the locked root toolchain + timeout-minutes: 5 + if: steps.stage.outputs.go == 'true' + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # The guardrails hooks read every Bash command through shfmt and refuse the + # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin + # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run + # replaces HOME. + - name: Install pinned shfmt for guardrails + if: steps.stage.outputs.go == 'true' && contains(steps.plugins.outputs.list, 'plugins/guardrails') + env: + SHFMT_VERSION: v3.14.1 + SHFMT_SHA256: 76e77641faa025814b77f153b29796b8e6fa2fca03e0c76a691608b86c7ea7bf + run: | + mkdir -p /opt/tools/bin + curl -fSL --no-progress-meter --retry 3 --retry-all-errors -o /opt/tools/bin/shfmt \ + "https://github.com/mvdan/sh/releases/download/$SHFMT_VERSION/shfmt_${SHFMT_VERSION}_linux_amd64" + echo "$SHFMT_SHA256 /opt/tools/bin/shfmt" | sha256sum -c - + chmod +x /opt/tools/bin/shfmt + echo "/opt/tools/bin" >> "$GITHUB_PATH" + test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # Proof that the managed policy holds before any pull request case runs, + # for each of the three tools a case can use to read a file: one Claude + # Code run is told to Read /proc/self/environ and the runner's .runner + # file, Grep /proc/self/environ, and Glob for environ under /proc/self. The job stops + # unless the result's permission_denials (code.claude.com/docs/en/ + # agent-sdk/typescript, "SDKResultMessage", as of 2026-10-28) records + # each of the four calls as denied, or if the result holds the + # credential anywhere. Claude Code applies Read deny rules to Glob and + # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, + # "Read & Edit"), so a Glob or Grep the policy does not stop fails here + # instead of reaching a case. The stream can hold what a call returned, + # so it stays in a file that is never printed and is removed after the + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. + - name: Probe that the policy denies those reads + id: probe + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no pull request case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + echo "The managed policy denied the probe's Read, Grep and Glob calls." + + # Chromium comes from the playwright that the locked playwright-cli + # and Playwright MCP server share. Without sudo there is no --with-deps: the hosted image already + # carries a browser's system libraries. A pull request run grants no + # Bash, so it needs no browser: it passes no --allow-tools, a case + # cannot widen that grant (plugins/evals/skills/plugin-eval/SKILL.md, + # the read-only suites paragraph), and select-plugins.mjs refuses a + # changed case's allowed_tools. + - name: Install Chromium + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + env: + PLAYWRIGHT_BROWSERS_PATH: /opt/pw + run: | + node /opt/tools/node_modules/playwright/cli.js install chromium + command -v playwright-cli claude + + # Every Bash call in an eval run goes through Claude Code's bubblewrap + # sandbox (code.claude.com/docs/en/sandboxing, "Set up Linux and WSL2", + # as of 2026-10-10). Starting bwrap here as the runner user, with the + # whole filesystem (so /opt/tools and the setup-node toolcache) bound + # read-only, fails the job with bwrap's own error if bwrap cannot start + # a sandbox, before any eval spends tokens. Its flags are not Claude + # Code's own, so a pass does not prove the full sandbox starts. + - name: Check bwrap starts a sandbox + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + run: | + fail() { + local msg=${2//'%'/'%25'} + msg=${msg//$'\r'/'%0D'} + echo "::error::$1: ${msg//$'\n'/'%0A'}" + exit 1 + } + for tool in bwrap socat; do + command -v "$tool" || fail "$tool is not on PATH" "the Bash sandbox needs it" + done + sysctl kernel.apparmor_restrict_unprivileged_userns || true + sandbox=(bwrap --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --unshare-all --die-with-parent) + if ! out=$("${sandbox[@]}" true 2>&1); then + fail "bwrap cannot start a sandbox as the runner user" "$out" + fi + if ! out=$("${sandbox[@]}" playwright-cli --version 2>&1); then + fail "playwright-cli does not start inside bwrap" "$out" + fi + echo "playwright-cli inside bwrap: $out" + + - name: Record that env scrub is off + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && !inputs.env-scrub + env: + DISPATCHER: ${{ github.actor }} + run: echo "::warning::CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is off for this run; Bash subprocesses can see the Claude credentials. Dispatched by ${DISPATCHER}." + + # One `claude plugin eval` per staged suite, each under its own + # --max-cost-usd. Its exit code is overloaded, so the summary step reads + # each JSON result instead. --keep-temp keeps each run's trace on the + # runner, outside the uploaded results, for the summary's health lines. + # A pull request run grants no tool, runs no scaffold, keeps env scrub on + # and uses each case's own run count. TMPDIR sits outside /tmp, which the + # run's own sandbox makes read-only to Bash, and within the short length + # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). + - name: Run the plugin eval + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + # Strips credentials from Bash subprocesses inside each eval run. + # The env-scrub input set false gives "0", the opt-out value Claude + # Code's own messages name. + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: ${{ (github.event_name == 'pull_request' || inputs.env-scrub) && '1' || '0' }} + SUITES: ${{ steps.stage.outputs.suites }} + DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} + CASE: ${{ inputs.case }} + TAGS: ${{ inputs.tag }} + RUNS: ${{ inputs.runs }} + ALLOW_TOOLS: ${{ inputs.allow-tools }} + REAL_SERVERS: ${{ inputs.allow-real-servers }} + OUT: ${{ runner.temp }}/eval-results + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" + args=( + --trust-plugin + --no-publish + --model claude-sonnet-5-5 + --judge-model claude-opus-5-5 + --max-cost-usd 5 + --keep-temp + ) + if [ "$DISPATCH" = true ]; then + if ! [[ $RUNS =~ ^[1-9][0-9]*$ ]]; then + echo "::error::runs must be a positive integer" + exit 1 + fi + IFS=, read -ra tools <<< "$ALLOW_TOOLS" + read -ra tags <<< "$TAGS" + for value in "${tools[@]}" "${tags[@]}" "$CASE"; do + case "$value" in + -*) echo "::error::a case, tag or tool must not start with -"; exit 1 ;; + esac + done + args+=(--scaffold --runs "$RUNS") + if [ "$REAL_SERVERS" = true ]; then + args+=(--allow-real-servers) + fi + if [ "${#tools[@]}" -gt 0 ]; then + args+=(--allow-tools "${tools[@]}") + fi + if [ "${#tags[@]}" -gt 0 ]; then + args+=(--tag "${tags[@]}") + fi + if [ -n "$CASE" ]; then + args+=(--case "$CASE") + fi + fi + while IFS=$'\t' read -r dir eval_dir; do + [ -n "$dir" ] || continue + out="$OUT/$(basename "$dir")" + mkdir -p "$out" + echo "== claude plugin eval $dir --eval-dir $eval_dir" + status=0 + if [ "$DISPATCH" = true ]; then + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" || status=$? + else + # A pull request run's console output can carry model text from + # the PR's cases, and the job log is public: it goes to a file + # that nothing uploads. + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" >"$out/console.log" 2>&1 || status=$? + fi + echo "exit $status" >"$out/exit-code" + done <<<"$SUITES" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT + # COMPARABLE per case. A suite with no result fails the job too. + # A pull request run prints no case name, error or partial reason: the + # model output those can carry would land in the public job summary. + # The REST API returns no job summary, so each suite's summary is also + # printed to the job log, with workflow commands off, and kept as + # summary.md beside its results.json, which a dispatched run uploads. + - name: Summarize the results + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + REDACT: ${{ github.event_name == 'pull_request' && '--redact' || '' }} + DISPATCHED: ${{ github.event_name == 'workflow_dispatch' }} + RUNS: ${{ inputs.runs }} + run: | + failed=0 + redact=() + if [ -n "$REDACT" ]; then redact=("$REDACT"); fi + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + name=$(basename "$dir") + results="$OUT/$name/results.json" + if [ ! -f "$results" ]; then + why=$(cat "$OUT/$name/exit-code" 2>/dev/null || echo "the run did not start") + printf '### Plugin eval: %s\n\nNo result was written (%s).\n\nResult: NOT COMPARABLE\n\n' \ + "$name" "$why" >>"$GITHUB_STEP_SUMMARY" + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::No result was written (${why//[$'\r\n%']/ })." + failed=1 + continue + fi + summary="$OUT/$name/summary.md" + annotation="$OUT/$name/annotation.txt" + node .base/.github/scripts/plugin-eval/summarize.mjs "$results" --plugin "$name" "${redact[@]}" --annotation "$annotation" >"$summary" || failed=1 + # A dispatched run uploads its results, but not the traces --keep-temp + # keeps here, so the validity verdict is read here and only its + # count-only bash ran line (the line just before the verdict, in its exact format) and its last line are kept (its detail + # lines can quote a trace), as eval-hill-climb-pass.yml does. + # noise-report reads only results.json, which the upload already + # holds. A dispatched run always scaffolds. + if [ "$DISPATCHED" = true ]; then + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$results" --runs "$RUNS" --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + verdict=$(printf '%s\n' "$validity" | tail -n 1) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$verdict" >"$OUT/$name/validity.txt" + python3 .base/plugins/evals/skills/plugin-eval/scripts/noise-report.py "$results" >"$OUT/$name/noise.txt" 2>&1 || true + printf '\nValidity: %s\n' "$verdict" >>"$summary" + if [ -n "$bash_ran" ]; then printf '\n%s\n' "$bash_ran" >>"$summary"; fi + fi + cat "$summary" >>"$GITHUB_STEP_SUMMARY" + echo >>"$GITHUB_STEP_SUMMARY" + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + cat "$summary" + echo "::$token::" + # summarize.mjs escapes the command it writes; it prints outside + # the stop-commands block so the runner reads it. + # No file at all: summarize.mjs could not read the result (exit 2). + if [ -s "$annotation" ]; then + head -n 1 "$annotation" + elif [ ! -e "$annotation" ]; then + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::The result could not be read." + fi + done <<<"$SUITES" + exit "$failed" + + # Opt-in because the job log of this public repository is public. Prints, + # per run, at most 20 Bash calls whose tool_result has is_error true: the + # command cut to 300 characters, and the result text cut to its first 120 + # and last 2500 characters when longer than 2620 (the end holds the daemon + # log), each JSON-quoted onto one line. A second line, `key:`, holds the lines of the + # full result that match FATAL, Check failed, ERROR:, sandbox, permission or errno + # patterns (the cause line that a long crash stack pushes out of the tail), minus the + # known .bashrc noise, de-duplicated, at most 15, each cut to 300 characters; it is + # omitted when nothing matches. Nothing else from the trace is printed. Trace text is + # tool output, so workflow commands are off while it prints. + - name: Print Bash error excerpts + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + OUT: ${{ runner.temp }}/eval-results + run: | + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + for RESULTS in "$OUT"/*/results.json; do + if [ ! -f "$RESULTS" ]; then + echo "no results.json, so no traces to read" + continue + fi + jq -r '.cases[]? as $c | $c.arms // {} | to_entries[] | .key as $arm + | .value // [] | to_entries[] + | [($c.name // $c.dir // "case"), $arm, (.key + 1 | tostring), (.value.tracePath // "")] + | @tsv' "$RESULTS" | + while IFS=$'\t' read -r name arm run trace; do + echo "== $name, $arm arm, run $run" + if [ ! -f "$trace" ]; then + echo " trace not found" + continue + fi + jq -nrR ' + def tail_cut: if length > 2620 then .[:120] + " โ€ฆ " + .[-2500:] else . end; + def key_lines: + [split("\n")[] + | select(test("FATAL|Check failed|ERROR:|No usable sandbox|Permission denied|Operation not permitted|EACCES|EPERM|ENOENT|errno") + and (contains(".bashrc: Permission denied") | not)) + | .[:300]] + | reduce .[] as $l ([]; if index([$l]) then . else . + [$l] end) + | .[:15] | join("\n"); + [inputs | fromjson? | objects] as $m + | ([$m[] | select(.type == "assistant") | .message.content[]? | objects + | select(.type == "tool_use" and .name == "Bash" and (.id | type) == "string") + | {key: .id, value: (.input.command // "" | tostring)}] | from_entries) as $bash + | [$m[] | select(.type == "user") | .message.content[]? | objects + | select(.type == "tool_result" and .is_error == true) + | (.tool_use_id // "" | tostring) as $id + | select($bash | has($id)) + | {cmd: $bash[$id], + out: (.content | if type == "array" then map(objects | select(.type == "text") | .text | tostring) | join("\n") else tostring end)}] + | " \(length) Bash error(s), showing up to 20", + (.[:20][] | " $ \(.cmd[:300] | tojson)\n -> \(.out | tail_cut | tojson)\(.out | key_lines | if . == "" then "" else "\n key: \(tojson)" end)") + ' "$trace" || echo " could not read the trace" + done || echo "could not read $RESULTS" + done + echo "::$token::" + + # Only a dispatched run, whose cases merged through review, uploads its + # results: a pull request's prompts could steer model output into a + # public artifact. + - name: Scan the upload for credentials + id: scan-upload + if: >- + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/eval-results/ + + - name: Upload the eval results + if: >- + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/eval-results/ + if-no-files-found: warn + retention-days: 3 + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.eval.outputs.handoff-reason || needs.eval-key-credit.outputs.skip-key == 'true' && 'credit-stop' || needs.eval.outputs.credential-present == 'false' && 'no-key' || '' }} + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + + # The fallback leg: the same steps on the central fallback token, after + # the subscription leg handed off or found its token empty, and only when + # the fallback token is set. It is the last leg and never hands off. + eval-fallback: + needs: [eval-key-credit, eval-subscription] + if: >- + ${{ !cancelled() && needs.eval-subscription.outputs.fallback-ready == 'true' + && (needs.eval-subscription.outputs.handoff == 'true' || needs.eval-subscription.outputs.credential-present == 'false') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the repository; nothing in this job writes + pull-requests: read # check-trusted-trigger reads the pull request + actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + API_BUDGET_TOKEN: github-token + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + # npm's fetch timeout and retries, from the one file that sets them. + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Resolve main's tip + id: tip + if: github.event_name == 'pull_request' + env: + TIP_BRANCH: ${{ github.event.repository.default_branch }} + GIT_TERMINAL_PROMPT: "0" + run: | + sha=$(timeout 60 git -C "$RUNNER_TEMP" ls-remote "https://github.com/$GITHUB_REPOSITORY.git" "refs/heads/$TIP_BRANCH" | + awk -v ref="refs/heads/$TIP_BRANCH" '$2 == ref { print $1 }') + [[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "::error::the tip of $TIP_BRANCH did not resolve"; exit 1; } + echo "sha=$sha" >>"$GITHUB_OUTPUT" + # The default branch tip; on a dispatch, the dispatched commit, which the + # job if: holds to the default branch. + - name: Check out the default branch + if: github.event_name != 'pull_request' || steps.tip.outputs.sha != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'pull_request' && steps.tip.outputs.sha || github.sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + # Skips the run while Actions or npm installs report a degraded component, + # with a logged reason; fails open (.github/actions/check-upstream-status). + - name: Check upstream status + id: upstream + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + timeout-minutes: 2 + uses: ./.base/.github/actions/check-upstream-status # zizmor: ignore[self-repository] + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + - name: Stop while upstream is degraded + if: steps.upstream.outputs.degraded == 'true' + env: + REASON: ${{ steps.upstream.outputs.reason }} + run: | + echo "::error::upstream-degraded: ${REASON} on its status page stops this run; dispatch it again once it recovers" + exit 1 + + # check-trusted-trigger needs a PR, and a dispatch names none, so the + # dispatcher is matched against the same list directly, by numeric id: + # it must be a listed human, and the lanes App bot is denied by id, as + # in pr-run-activity-write.yml. A re-run is refused: its re-runner's id + # is only in the run's record, which this job's token cannot read, and + # a login can pass to another account. Dispatch the workflow again + # instead. + - name: Check the dispatcher is a trusted human + if: github.event_name == 'workflow_dispatch' + env: + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + ACTOR_ID: ${{ github.actor_id }} + APP_SENDER_ID: ${{ vars.AUTOMATION_LANES_APP_SENDER_ID }} + run: | + if [ "$GITHUB_RUN_ATTEMPT" != 1 ]; then + echo "::error::rerun-refused: a re-run of this workflow is not supported; dispatch it again" + exit 1 + fi + if ! [[ $APP_SENDER_ID =~ ^[1-9][0-9]*$ ]]; then + echo "::error::the AUTOMATION_LANES_APP_SENDER_ID variable must hold the lanes App bot's numeric user id" + exit 1 + fi + if [ "$ACTOR_ID" = "$APP_SENDER_ID" ]; then + echo "::error::bot-actor: the lanes App bot cannot dispatch this lane" + exit 1 + fi + if ! jq -e --argjson id "$ACTOR_ID" ' + .version == 1 + and any(.actors[]; .id == $id and .kind == "human") + ' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the dispatcher is not a listed human" + exit 1 + fi + + - name: Check the pull request trigger + id: gate + if: github.event_name == 'pull_request' + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + + # check-trusted-trigger accepts any listed actor; the label that spends + # the credential must come from a listed human, so a person opts in to + # each run. + - name: Stop on an untrusted pull request trigger + if: github.event_name == 'pull_request' + env: + PROCEED: ${{ steps.gate.outputs.proceed }} + REASON: ${{ steps.gate.outputs.reason }} + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + SENDER_ID: ${{ github.event.sender.id }} + run: | + if [ "$PROCEED" != true ]; then + echo "::error::the trigger is not trusted (reason: ${REASON:-unreadable})" + exit 1 + fi + if ! jq -e --argjson id "$SENDER_ID" 'any(.actors[]; .id == $id and .kind == "human")' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the 'eval: run' label did not come from a listed human" + exit 1 + fi + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + echo "::notice::FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is empty; no eval runs." + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; } >>"$GITHUB_OUTPUT" + + # The plugin under test on a pull request: the label-time head, with full + # history so the next step diffs this same commit against its base. A + # checkout runs no code from it. + - name: Check out the pull request head + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: .head + fetch-depth: 0 + persist-credentials: false + + # The checked-out head's diff from its merge base with the pull + # request's base chooses the plugins; the target input does on a + # dispatch. Each line of `plugins` is a plugins/ path. + - name: Choose the plugins + id: plugins + if: steps.credential.outputs.run == 'true' + env: + EVENT: ${{ github.event_name }} + TARGET: ${{ inputs.target }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ "$EVENT" = workflow_dispatch ]; then + target=${TARGET%/} + if ! [[ $target =~ ^plugins/[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::target must be a plugin directory, plugins/" + exit 1 + fi + selected=$target + else + merge_base=$(git -C .head merge-base "$BASE_SHA" "$HEAD_SHA") + files=$(git -C .head diff --no-renames --name-only "$merge_base" "$HEAD_SHA") + selected=$(printf '%s\n' "$files" | + node .base/.github/scripts/plugin-eval/select-plugins.mjs --repo .head --base "$merge_base" --head "$HEAD_SHA") || { + echo "::error::the pull request changes more of a selected plugin than its suite may run with the credential" + exit 1 + } + fi + if [ -z "$selected" ]; then + echo "::notice::the pull request changes no plugin's skills, agents or evals; no eval runs." + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "list<>"$GITHUB_OUTPUT" + + # validate-cases.py makes no model call. A FAIL is a case the binary + # rejects at load, so the job stops before any spend. On a deterministic + # run each suite is then copied without its judged cases. + - name: Validate and stage the suites + id: stage + if: steps.plugins.outputs.go == 'true' + env: + LIST: ${{ steps.plugins.outputs.list }} + ROOT: ${{ github.event_name == 'pull_request' && '.head' || '.base' }} + DETERMINISTIC: ${{ github.event_name == 'pull_request' || inputs.deterministic-only }} + run: | + staged="" + while IFS= read -r plugin; do + [ -n "$plugin" ] || continue + dir="$ROOT/$plugin" + eval_dir=$(jq -r '.experimental.evals // "evals"' "$dir/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + if [ ! -d "$dir/$eval_dir" ]; then + echo "::notice::$plugin has no eval suite; skipped." + continue + fi + echo "== validate $plugin/$eval_dir" + if ! python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$dir/$eval_dir"; then + echo "::error::$plugin/$eval_dir does not pass validate-cases.py, so no eval runs" + exit 1 + fi + if [ "$DETERMINISTIC" = true ]; then + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$dir" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::notice::$plugin has no case with deterministic graders only; skipped." + continue + fi + eval_dir=evals-deterministic + fi + staged+="$dir"$'\t'"$eval_dir"$'\n' + done <<<"$LIST" + if [ -z "$staged" ]; then + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "suites<>"$GITHUB_OUTPUT" + + # The tools and Chromium go under /opt, outside HOME, which each eval + # run replaces with a temporary one. Only these directories are made + # while sudo works; no third-party code runs before isolation. + - name: Create the tool directories + if: steps.stage.outputs.go == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools /opt/pw + + # A pull request supplies the prompts, and the eval's Read, Glob and Grep + # run without the Bash sandbox, so nothing else keeps them from the + # credential in /proc//environ or the runner's own files. A + # root-owned managed settings file, which applies inside every eval run + # and which nothing in the run overrides (code.claude.com/docs/en/ + # plugin-evals, "How runs are isolated"; code.claude.com/docs/en/ + # settings, "Settings precedence", as of 2026-10-27), denies those + # paths. The runner's install directory is read from the Runner.Worker + # process above this step, not guessed; the job's _temp and _actions + # directories are denied beside it. Written while sudo is still there; + # the job stops if the runner directory is not found or the file cannot + # be written or read back. The probe step below proves the denial. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions." + + # Installs the bubblewrap and socat the Bash sandbox needs, lifts the + # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo + # and docker access for the rest of the job. + - name: Isolate the runner + if: steps.stage.outputs.go == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + # Every binary this job runs comes from the locked root toolchain. Its + # install scripts run here, after sudo and docker access are gone. + - name: Install the locked root toolchain + timeout-minutes: 5 + if: steps.stage.outputs.go == 'true' + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # The guardrails hooks read every Bash command through shfmt and refuse the + # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin + # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run + # replaces HOME. + - name: Install pinned shfmt for guardrails + if: steps.stage.outputs.go == 'true' && contains(steps.plugins.outputs.list, 'plugins/guardrails') + env: + SHFMT_VERSION: v3.14.1 + SHFMT_SHA256: 76e77641faa025814b77f153b29796b8e6fa2fca03e0c76a691608b86c7ea7bf + run: | + mkdir -p /opt/tools/bin + curl -fSL --no-progress-meter --retry 3 --retry-all-errors -o /opt/tools/bin/shfmt \ + "https://github.com/mvdan/sh/releases/download/$SHFMT_VERSION/shfmt_${SHFMT_VERSION}_linux_amd64" + echo "$SHFMT_SHA256 /opt/tools/bin/shfmt" | sha256sum -c - + chmod +x /opt/tools/bin/shfmt + echo "/opt/tools/bin" >> "$GITHUB_PATH" + test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # Proof that the managed policy holds before any pull request case runs, + # for each of the three tools a case can use to read a file: one Claude + # Code run is told to Read /proc/self/environ and the runner's .runner + # file, Grep /proc/self/environ, and Glob for environ under /proc/self. The job stops + # unless the result's permission_denials (code.claude.com/docs/en/ + # agent-sdk/typescript, "SDKResultMessage", as of 2026-10-28) records + # each of the four calls as denied, or if the result holds the + # credential anywhere. Claude Code applies Read deny rules to Glob and + # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, + # "Read & Edit"), so a Glob or Grep the policy does not stop fails here + # instead of reaching a case. The stream can hold what a call returned, + # so it stays in a file that is never printed and is removed after the + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. + - name: Probe that the policy denies those reads + id: probe + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no pull request case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + echo "The managed policy denied the probe's Read, Grep and Glob calls." + + # Chromium comes from the playwright that the locked playwright-cli + # and Playwright MCP server share. Without sudo there is no --with-deps: the hosted image already + # carries a browser's system libraries. A pull request run grants no + # Bash, so it needs no browser: it passes no --allow-tools, a case + # cannot widen that grant (plugins/evals/skills/plugin-eval/SKILL.md, + # the read-only suites paragraph), and select-plugins.mjs refuses a + # changed case's allowed_tools. + - name: Install Chromium + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + env: + PLAYWRIGHT_BROWSERS_PATH: /opt/pw + run: | + node /opt/tools/node_modules/playwright/cli.js install chromium + command -v playwright-cli claude + + # Every Bash call in an eval run goes through Claude Code's bubblewrap + # sandbox (code.claude.com/docs/en/sandboxing, "Set up Linux and WSL2", + # as of 2026-10-10). Starting bwrap here as the runner user, with the + # whole filesystem (so /opt/tools and the setup-node toolcache) bound + # read-only, fails the job with bwrap's own error if bwrap cannot start + # a sandbox, before any eval spends tokens. Its flags are not Claude + # Code's own, so a pass does not prove the full sandbox starts. + - name: Check bwrap starts a sandbox + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + run: | + fail() { + local msg=${2//'%'/'%25'} + msg=${msg//$'\r'/'%0D'} + echo "::error::$1: ${msg//$'\n'/'%0A'}" + exit 1 + } + for tool in bwrap socat; do + command -v "$tool" || fail "$tool is not on PATH" "the Bash sandbox needs it" + done + sysctl kernel.apparmor_restrict_unprivileged_userns || true + sandbox=(bwrap --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --unshare-all --die-with-parent) + if ! out=$("${sandbox[@]}" true 2>&1); then + fail "bwrap cannot start a sandbox as the runner user" "$out" + fi + if ! out=$("${sandbox[@]}" playwright-cli --version 2>&1); then + fail "playwright-cli does not start inside bwrap" "$out" + fi + echo "playwright-cli inside bwrap: $out" + + - name: Record that env scrub is off + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && !inputs.env-scrub + env: + DISPATCHER: ${{ github.actor }} + run: echo "::warning::CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is off for this run; Bash subprocesses can see the Claude credentials. Dispatched by ${DISPATCHER}." + + # One `claude plugin eval` per staged suite, each under its own + # --max-cost-usd. Its exit code is overloaded, so the summary step reads + # each JSON result instead. --keep-temp keeps each run's trace on the + # runner, outside the uploaded results, for the summary's health lines. + # A pull request run grants no tool, runs no scaffold, keeps env scrub on + # and uses each case's own run count. TMPDIR sits outside /tmp, which the + # run's own sandbox makes read-only to Bash, and within the short length + # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). + - name: Run the plugin eval + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + # Strips credentials from Bash subprocesses inside each eval run. + # The env-scrub input set false gives "0", the opt-out value Claude + # Code's own messages name. + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: ${{ (github.event_name == 'pull_request' || inputs.env-scrub) && '1' || '0' }} + SUITES: ${{ steps.stage.outputs.suites }} + DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} + CASE: ${{ inputs.case }} + TAGS: ${{ inputs.tag }} + RUNS: ${{ inputs.runs }} + ALLOW_TOOLS: ${{ inputs.allow-tools }} + REAL_SERVERS: ${{ inputs.allow-real-servers }} + OUT: ${{ runner.temp }}/eval-results + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" + args=( + --trust-plugin + --no-publish + --model claude-sonnet-5-5 + --judge-model claude-opus-5-5 + --max-cost-usd 5 + --keep-temp + ) + if [ "$DISPATCH" = true ]; then + if ! [[ $RUNS =~ ^[1-9][0-9]*$ ]]; then + echo "::error::runs must be a positive integer" + exit 1 + fi + IFS=, read -ra tools <<< "$ALLOW_TOOLS" + read -ra tags <<< "$TAGS" + for value in "${tools[@]}" "${tags[@]}" "$CASE"; do + case "$value" in + -*) echo "::error::a case, tag or tool must not start with -"; exit 1 ;; + esac + done + args+=(--scaffold --runs "$RUNS") + if [ "$REAL_SERVERS" = true ]; then + args+=(--allow-real-servers) + fi + if [ "${#tools[@]}" -gt 0 ]; then + args+=(--allow-tools "${tools[@]}") + fi + if [ "${#tags[@]}" -gt 0 ]; then + args+=(--tag "${tags[@]}") + fi + if [ -n "$CASE" ]; then + args+=(--case "$CASE") + fi + fi + while IFS=$'\t' read -r dir eval_dir; do + [ -n "$dir" ] || continue + out="$OUT/$(basename "$dir")" + mkdir -p "$out" + echo "== claude plugin eval $dir --eval-dir $eval_dir" + status=0 + if [ "$DISPATCH" = true ]; then + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" || status=$? + else + # A pull request run's console output can carry model text from + # the PR's cases, and the job log is public: it goes to a file + # that nothing uploads. + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" >"$out/console.log" 2>&1 || status=$? + fi + echo "exit $status" >"$out/exit-code" + done <<<"$SUITES" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT + # COMPARABLE per case. A suite with no result fails the job too. + # A pull request run prints no case name, error or partial reason: the + # model output those can carry would land in the public job summary. + # The REST API returns no job summary, so each suite's summary is also + # printed to the job log, with workflow commands off, and kept as + # summary.md beside its results.json, which a dispatched run uploads. + - name: Summarize the results + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + REDACT: ${{ github.event_name == 'pull_request' && '--redact' || '' }} + DISPATCHED: ${{ github.event_name == 'workflow_dispatch' }} + RUNS: ${{ inputs.runs }} + run: | + failed=0 + redact=() + if [ -n "$REDACT" ]; then redact=("$REDACT"); fi + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + name=$(basename "$dir") + results="$OUT/$name/results.json" + if [ ! -f "$results" ]; then + why=$(cat "$OUT/$name/exit-code" 2>/dev/null || echo "the run did not start") + printf '### Plugin eval: %s\n\nNo result was written (%s).\n\nResult: NOT COMPARABLE\n\n' \ + "$name" "$why" >>"$GITHUB_STEP_SUMMARY" + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::No result was written (${why//[$'\r\n%']/ })." + failed=1 + continue + fi + summary="$OUT/$name/summary.md" + annotation="$OUT/$name/annotation.txt" + node .base/.github/scripts/plugin-eval/summarize.mjs "$results" --plugin "$name" "${redact[@]}" --annotation "$annotation" >"$summary" || failed=1 + # A dispatched run uploads its results, but not the traces --keep-temp + # keeps here, so the validity verdict is read here and only its + # count-only bash ran line (the line just before the verdict, in its exact format) and its last line are kept (its detail + # lines can quote a trace), as eval-hill-climb-pass.yml does. + # noise-report reads only results.json, which the upload already + # holds. A dispatched run always scaffolds. + if [ "$DISPATCHED" = true ]; then + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$results" --runs "$RUNS" --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + verdict=$(printf '%s\n' "$validity" | tail -n 1) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$verdict" >"$OUT/$name/validity.txt" + python3 .base/plugins/evals/skills/plugin-eval/scripts/noise-report.py "$results" >"$OUT/$name/noise.txt" 2>&1 || true + printf '\nValidity: %s\n' "$verdict" >>"$summary" + if [ -n "$bash_ran" ]; then printf '\n%s\n' "$bash_ran" >>"$summary"; fi + fi + cat "$summary" >>"$GITHUB_STEP_SUMMARY" + echo >>"$GITHUB_STEP_SUMMARY" + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + cat "$summary" + echo "::$token::" + # summarize.mjs escapes the command it writes; it prints outside + # the stop-commands block so the runner reads it. + # No file at all: summarize.mjs could not read the result (exit 2). + if [ -s "$annotation" ]; then + head -n 1 "$annotation" + elif [ ! -e "$annotation" ]; then + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::The result could not be read." + fi + done <<<"$SUITES" + exit "$failed" + + # Opt-in because the job log of this public repository is public. Prints, + # per run, at most 20 Bash calls whose tool_result has is_error true: the + # command cut to 300 characters, and the result text cut to its first 120 + # and last 2500 characters when longer than 2620 (the end holds the daemon + # log), each JSON-quoted onto one line. A second line, `key:`, holds the lines of the + # full result that match FATAL, Check failed, ERROR:, sandbox, permission or errno + # patterns (the cause line that a long crash stack pushes out of the tail), minus the + # known .bashrc noise, de-duplicated, at most 15, each cut to 300 characters; it is + # omitted when nothing matches. Nothing else from the trace is printed. Trace text is + # tool output, so workflow commands are off while it prints. + - name: Print Bash error excerpts + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + OUT: ${{ runner.temp }}/eval-results + run: | + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + for RESULTS in "$OUT"/*/results.json; do + if [ ! -f "$RESULTS" ]; then + echo "no results.json, so no traces to read" + continue + fi + jq -r '.cases[]? as $c | $c.arms // {} | to_entries[] | .key as $arm + | .value // [] | to_entries[] + | [($c.name // $c.dir // "case"), $arm, (.key + 1 | tostring), (.value.tracePath // "")] + | @tsv' "$RESULTS" | + while IFS=$'\t' read -r name arm run trace; do + echo "== $name, $arm arm, run $run" + if [ ! -f "$trace" ]; then + echo " trace not found" + continue + fi + jq -nrR ' + def tail_cut: if length > 2620 then .[:120] + " โ€ฆ " + .[-2500:] else . end; + def key_lines: + [split("\n")[] + | select(test("FATAL|Check failed|ERROR:|No usable sandbox|Permission denied|Operation not permitted|EACCES|EPERM|ENOENT|errno") + and (contains(".bashrc: Permission denied") | not)) + | .[:300]] + | reduce .[] as $l ([]; if index([$l]) then . else . + [$l] end) + | .[:15] | join("\n"); + [inputs | fromjson? | objects] as $m + | ([$m[] | select(.type == "assistant") | .message.content[]? | objects + | select(.type == "tool_use" and .name == "Bash" and (.id | type) == "string") + | {key: .id, value: (.input.command // "" | tostring)}] | from_entries) as $bash + | [$m[] | select(.type == "user") | .message.content[]? | objects + | select(.type == "tool_result" and .is_error == true) + | (.tool_use_id // "" | tostring) as $id + | select($bash | has($id)) + | {cmd: $bash[$id], + out: (.content | if type == "array" then map(objects | select(.type == "text") | .text | tostring) | join("\n") else tostring end)}] + | " \(length) Bash error(s), showing up to 20", + (.[:20][] | " $ \(.cmd[:300] | tojson)\n -> \(.out | tail_cut | tojson)\(.out | key_lines | if . == "" then "" else "\n key: \(tojson)" end)") + ' "$trace" || echo " could not read the trace" + done || echo "could not read $RESULTS" + done + echo "::$token::" + + # Only a dispatched run, whose cases merged through review, uploads its + # results: a pull request's prompts could steer model output into a + # public artifact. + - name: Scan the upload for credentials + id: scan-upload + if: >- + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/eval-results/ + + - name: Upload the eval results + if: >- + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/eval-results/ + if-no-files-found: warn + retention-days: 3 + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: FALLBACK_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.eval-subscription.outputs.handoff-reason || 'no-key' }} + - name: Log the API budget at the end if: always() continue-on-error: true diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml index b251d86bc6c..244615f7698 100644 --- a/.github/workflows/eval-hill-climb-pass.yml +++ b/.github/workflows/eval-hill-climb-pass.yml @@ -14,6 +14,18 @@ name: eval-hill-climb-pass # tool is granted beyond the read-only set. Console output, which can carry # model text, goes to a file that nothing uploads. # +# The pass runs on the maintenance role's credential chain, one credential +# per job: `pass` on MAINTENANCE_CLAUDE_API_KEY, then `pass-subscription` on +# MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN, then `pass-fallback` on +# FALLBACK_CLAUDE_CODE_OAUTH_TOKEN. A leg hands off on the stops run-claude-cli +# hands off on (eval-handoff.mjs reads the probe's stream and every errored +# run's kept trace with run-claude-cli's own classifier): the key leg on a +# credit stop or a 401 or 402, the subscription leg, when the fallback token +# is set, also on a limit stop; an empty credential hands off too. A leg that +# hands off uploads nothing, so the pass's outputs are the one leg that +# finished. The gate job skips the key after a recent credit stop +# (ci-workflows check-key-credit). +# # The pass uploads only what eval-browser-dispatch.yml uploads, its result # (per-case scores and grader verdicts), with summarize.mjs's summary (its # health lines read from the kept traces) and run-validity.py's bash ran and @@ -60,21 +72,72 @@ on: type: string default: "" secrets: + MAINTENANCE_CLAUDE_API_KEY: + description: The maintenance role's Console API key, the key leg's credential. + required: false MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: - description: The maintenance role's subscription token. - required: true + description: The maintenance role's subscription token, the subscription leg's credential. + required: false + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: + description: The central fallback subscription token, the fallback leg's credential. + required: false outputs: artifact-id: - description: The pass artifact's id. - value: ${{ jobs.pass.outputs.artifact-id }} + description: The pass artifact's id, from the leg that finished the pass. + value: ${{ jobs.pass-fallback.outputs.artifact-id || jobs.pass-subscription.outputs.artifact-id || jobs.pass.outputs.artifact-id }} manifest-sha256: - description: SHA-256 of the artifact's manifest.sha256. - value: ${{ jobs.pass.outputs.manifest-sha256 }} + description: SHA-256 of that artifact's manifest.sha256. + value: ${{ jobs.pass-fallback.outputs.manifest-sha256 || jobs.pass-subscription.outputs.manifest-sha256 || jobs.pass.outputs.manifest-sha256 }} permissions: {} jobs: + # The key's credit gate (ci-workflows check-key-credit, gate.cjs: one + # artifact-list call for the marker the key leg uploads after a credit + # stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour unset) + # starts the pass on the subscription, with no refused key attempt first. + # No secret, no model, no checkout; every failure runs the key (skip-key + # empty). Probing is off. + gate: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: pass + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" + + # The key leg: the pass on the maintenance role's API key alone. A credit + # stop, or a 401 or 402, in the probe or in any run hands the pass to + # pass-subscription, and this leg uploads nothing. An empty key, or one + # the gate skipped, runs nothing here. pass: + needs: gate + if: ${{ !cancelled() && needs.gate.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 60 cache-mode: read @@ -83,6 +146,10 @@ jobs: outputs: artifact-id: ${{ steps.upload.outputs.artifact-id }} manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} env: npm_config_audit: "false" npm_config_fetch_timeout: "60000" @@ -116,18 +183,25 @@ jobs: echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" exit 1 - # Only whether the secret is set leaves this step, never its value. + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. - name: Check the credential + id: credential env: - PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.gate.outputs.skip-key }} run: | - if [ "$PRESENT" != true ]; then - echo "::error::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty, so no case runs" - exit 1 + run=$PRESENT + if [ "$SKIP_KEY" = true ]; then + run=false + echo "::notice::the credit gate skipped MAINTENANCE_CLAUDE_API_KEY; pass-subscription runs the pass" + elif [ "$PRESENT" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; pass-subscription runs the pass" fi + { echo "present=$PRESENT"; echo "run=$run"; } >>"$GITHUB_OUTPUT" - name: Download the edit - if: inputs.pass == 'b' + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: artifact-ids: ${{ inputs.edit-artifact-id }} @@ -136,7 +210,7 @@ jobs: # The patch the scan job checked, byte for byte, applied with git's own # path checks; nothing else of the artifact is read. - name: Apply the edit - if: inputs.pass == 'b' + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' env: PATCH: ${{ runner.temp }}/edit/edit.patch WANT: ${{ inputs.edit-sha256 }} @@ -156,6 +230,7 @@ jobs: # validate-cases.py makes no model call; the suite is then copied without # its judged cases. - name: Stage the suite + if: steps.credential.outputs.run == 'true' env: DIR: .base/plugins/${{ inputs.plugin }} run: | @@ -169,6 +244,7 @@ jobs: fi - name: Create the tool directories + if: steps.credential.outputs.run == 'true' run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools # eval-browser-dispatch.yml's pull request policy, for the same reason: @@ -176,6 +252,7 @@ jobs: # runs model-written skill text. - name: Deny the eval's reads of process and runner state id: deny + if: steps.credential.outputs.run == 'true' run: | runner_root="" pid=$$ while [ "${pid:-0}" -gt 1 ]; do @@ -210,35 +287,74 @@ jobs: echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" - name: Isolate the runner + if: steps.credential.outputs.run == 'true' uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] with: install-subprocess-isolation: "true" - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' timeout-minutes: 5 run: | cp .base/package.json .base/package-lock.json /opt/tools/ npm ci --prefix /opt/tools echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + # eval-browser-dispatch.yml's probe: the job stops unless the result # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } out=$(mktemp) trap 'rm -f "$out"' EXIT + status=0 claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ - --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format json >"$out" 2>/dev/null || true - if grep -qF "$CLAUDE_CODE_OAUTH_TOKEN" "$out"; then + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi denied() { - jq -e --arg t "$1" --arg p "$2" 'any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } } denied Read /proc/self/environ @@ -249,14 +365,20 @@ jobs: # Identical flags on every pass. TMPDIR sits outside /tmp, as in # eval-browser-dispatch.yml. - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" DIR: .base/plugins/${{ inputs.plugin }} TAGS: ${{ inputs.tags }} CASE: ${{ inputs.case }} OUT: ${{ runner.temp }}/pass run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi export TMPDIR="$HOME/.ev" mkdir -p "$TMPDIR" "$OUT" args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 @@ -272,12 +394,37 @@ jobs: >"$RUNNER_TEMP/console.log" 2>&1 || status=$? echo "claude plugin eval exited $status" + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + # The summary (health lines from the kept traces) and the validity # verdict, read here because the traces stay on this runner. Only the # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's - # detail lines can quote a trace. + # detail lines can quote a trace. A leg that handed off reads nothing. - name: Read the pass id: read + if: steps.judge.outputs.handoff == 'false' env: OUT: ${{ runner.temp }}/pass PLUGIN: ${{ inputs.plugin }} @@ -302,16 +449,770 @@ jobs: - name: Scan the upload for credentials id: scan-upload + if: steps.read.outcome == 'success' uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] with: path: ${{ runner.temp }}/upload/ - name: Upload the pass id: upload - if: steps.scan-upload.outputs.clean == 'true' + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.run_id }} + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} path: ${{ runner.temp }}/upload/ retention-days: 1 if-no-files-found: error + + # Read by the next runs' credit gate: the artifact its `marker` output + # names (check-key-credit gate.cjs markerName), uploaded only after the + # key leg's credit-stop handoff. The name carries the caller workflow's + # file, since a called workflow's github context is the caller's, and is + # written out so its retention row claims it. pass-a and pass-a2 can both + # upload it in one run, so a failed upload never fails the job. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.gate.outputs.marker == 'key-credit-stop-maintenance-measure-skill-edit.yml-pass' && steps.judge.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + id: credit-marker-upload + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-maintenance-measure-skill-edit.yml-pass + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_API_KEY + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: "" + + # The subscription leg: the key leg's steps on a fresh runner, on the + # maintenance role's subscription token alone, with no credit-stop marker. + # With FALLBACK_CLAUDE_CODE_OAUTH_TOKEN set, a limit stop, a credit stop, + # or a 401 or 402 hands off to pass-fallback, and so does an empty token. + # hill-climb-workflow.test.mjs holds its steps equal to the key leg's but + # for the credential and the leg record's fallback reason. + pass-subscription: + needs: [gate, pass] + if: >- + ${{ !cancelled() && needs.gate.result == 'success' + && (needs.pass.outputs.handoff == 'true' || needs.pass.outputs.credential-present == 'false' + || needs.gate.outputs.skip-key == 'true') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the default branch; nothing in this job writes + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Check out the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.base-sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + if [ "$FALLBACK_PRESENT" != true ]; then + echo "::error::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are both empty, so no case runs" + exit 1 + fi + echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; pass-fallback runs the pass" + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" + + - name: Download the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ inputs.edit-artifact-id }} + path: ${{ runner.temp }}/edit + + # The patch the scan job checked, byte for byte, applied with git's own + # path checks; nothing else of the artifact is read. + - name: Apply the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + env: + PATCH: ${{ runner.temp }}/edit/edit.patch + WANT: ${{ inputs.edit-sha256 }} + run: | + if [ -L "$PATCH" ] || [ ! -f "$PATCH" ]; then + echo "::error::the edit artifact holds no edit.patch" + exit 1 + fi + sum=$(sha256sum <"$PATCH") + if ! [[ $WANT =~ ^[0-9a-f]{64}$ ]] || [ "${sum%% *}" != "$WANT" ]; then + echo "::error::edit.patch does not match the SHA-256 the scan job recorded" + exit 1 + fi + git -C .base apply --check "$PATCH" + git -C .base apply "$PATCH" + + # validate-cases.py makes no model call; the suite is then copied without + # its judged cases. + - name: Stage the suite + if: steps.credential.outputs.run == 'true' + env: + DIR: .base/plugins/${{ inputs.plugin }} + run: | + eval_dir=$(jq -r '.experimental.evals // "evals"' "$DIR/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$DIR/$eval_dir" + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$DIR" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::error::the suite has no case with deterministic graders only" + exit 1 + fi + + - name: Create the tool directories + if: steps.credential.outputs.run == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools + + # eval-browser-dispatch.yml's pull request policy, for the same reason: + # the eval's Read, Glob and Grep run without the Bash sandbox, and pass b + # runs model-written skill text. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.credential.outputs.run == 'true' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + + - name: Isolate the runner + if: steps.credential.outputs.run == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' + timeout-minutes: 5 + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # eval-browser-dispatch.yml's probe: the job stops unless the result + # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. + - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + + # Identical flags on every pass. TMPDIR sits outside /tmp, as in + # eval-browser-dispatch.yml. + - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + DIR: .base/plugins/${{ inputs.plugin }} + TAGS: ${{ inputs.tags }} + CASE: ${{ inputs.case }} + OUT: ${{ runner.temp }}/pass + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" "$OUT" + args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 + --max-cost-usd 5 --runs 3 --scaffold --keep-temp) + read -ra tags <<<"$TAGS" + for value in "${tags[@]}" "$CASE"; do + case "$value" in -*) echo "::error::a case or tag must not start with -"; exit 1 ;; esac + done + if [ "${#tags[@]}" -gt 0 ]; then args+=(--tag "${tags[@]}"); fi + if [ -n "$CASE" ]; then args+=(--case "$CASE"); fi + status=0 + claude plugin eval "$DIR" --eval-dir evals-deterministic --output-dir "$OUT/run" --json "$OUT/results.json" "${args[@]}" \ + >"$RUNNER_TEMP/console.log" 2>&1 || status=$? + echo "claude plugin eval exited $status" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The summary (health lines from the kept traces) and the validity + # verdict, read here because the traces stay on this runner. Only the + # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's + # detail lines can quote a trace. A leg that handed off reads nothing. + - name: Read the pass + id: read + if: steps.judge.outputs.handoff == 'false' + env: + OUT: ${{ runner.temp }}/pass + PLUGIN: ${{ inputs.plugin }} + PASS: ${{ inputs.pass }} + STAGE: ${{ runner.temp }}/upload + run: | + mkdir -p "$STAGE" + if [ ! -f "$OUT/results.json" ]; then + echo "::error::pass $PASS wrote no result" + exit 1 + fi + cp "$OUT/results.json" "$STAGE/results.json" + node .base/.github/scripts/plugin-eval/summarize.mjs "$OUT/results.json" --plugin "$PLUGIN" >"$STAGE/summary.md" || true + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$OUT/results.json" --runs 3 --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$(printf '%s\n' "$validity" | tail -n 1)" >"$STAGE/validity.txt" + cat "$STAGE/validity.txt" + sums=$(cd "$STAGE" && find . -type f -print0 | sort -z | xargs -0 sha256sum) + printf '%s\n' "$sums" >"$STAGE/manifest.sha256" + sum=$(sha256sum <"$STAGE/manifest.sha256") + echo "manifest-sha256=${sum%% *}" >>"$GITHUB_OUTPUT" + + - name: Scan the upload for credentials + id: scan-upload + if: steps.read.outcome == 'success' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/upload/ + + - name: Upload the pass + id: upload + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} + path: ${{ runner.temp }}/upload/ + retention-days: 1 + if-no-files-found: error + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.pass.outputs.handoff-reason || needs.gate.outputs.skip-key == 'true' && 'credit-stop' || needs.pass.outputs.credential-present == 'false' && 'no-key' || '' }} + + # The fallback leg: the same steps on the central fallback token, after + # the subscription leg handed off or found its token empty, and only when + # the fallback token is set. It is the last leg and never hands off. + pass-fallback: + needs: [gate, pass-subscription] + if: >- + ${{ !cancelled() && needs.pass-subscription.outputs.fallback-ready == 'true' + && (needs.pass-subscription.outputs.handoff == 'true' || needs.pass-subscription.outputs.credential-present == 'false') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the default branch; nothing in this job writes + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Check out the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.base-sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + echo "::error::FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is empty, so no case runs" + exit 1 + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; } >>"$GITHUB_OUTPUT" + + - name: Download the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ inputs.edit-artifact-id }} + path: ${{ runner.temp }}/edit + + # The patch the scan job checked, byte for byte, applied with git's own + # path checks; nothing else of the artifact is read. + - name: Apply the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + env: + PATCH: ${{ runner.temp }}/edit/edit.patch + WANT: ${{ inputs.edit-sha256 }} + run: | + if [ -L "$PATCH" ] || [ ! -f "$PATCH" ]; then + echo "::error::the edit artifact holds no edit.patch" + exit 1 + fi + sum=$(sha256sum <"$PATCH") + if ! [[ $WANT =~ ^[0-9a-f]{64}$ ]] || [ "${sum%% *}" != "$WANT" ]; then + echo "::error::edit.patch does not match the SHA-256 the scan job recorded" + exit 1 + fi + git -C .base apply --check "$PATCH" + git -C .base apply "$PATCH" + + # validate-cases.py makes no model call; the suite is then copied without + # its judged cases. + - name: Stage the suite + if: steps.credential.outputs.run == 'true' + env: + DIR: .base/plugins/${{ inputs.plugin }} + run: | + eval_dir=$(jq -r '.experimental.evals // "evals"' "$DIR/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$DIR/$eval_dir" + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$DIR" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::error::the suite has no case with deterministic graders only" + exit 1 + fi + + - name: Create the tool directories + if: steps.credential.outputs.run == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools + + # eval-browser-dispatch.yml's pull request policy, for the same reason: + # the eval's Read, Glob and Grep run without the Bash sandbox, and pass b + # runs model-written skill text. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.credential.outputs.run == 'true' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + + - name: Isolate the runner + if: steps.credential.outputs.run == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' + timeout-minutes: 5 + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # eval-browser-dispatch.yml's probe: the job stops unless the result + # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. + - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + + # Identical flags on every pass. TMPDIR sits outside /tmp, as in + # eval-browser-dispatch.yml. + - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + DIR: .base/plugins/${{ inputs.plugin }} + TAGS: ${{ inputs.tags }} + CASE: ${{ inputs.case }} + OUT: ${{ runner.temp }}/pass + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" "$OUT" + args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 + --max-cost-usd 5 --runs 3 --scaffold --keep-temp) + read -ra tags <<<"$TAGS" + for value in "${tags[@]}" "$CASE"; do + case "$value" in -*) echo "::error::a case or tag must not start with -"; exit 1 ;; esac + done + if [ "${#tags[@]}" -gt 0 ]; then args+=(--tag "${tags[@]}"); fi + if [ -n "$CASE" ]; then args+=(--case "$CASE"); fi + status=0 + claude plugin eval "$DIR" --eval-dir evals-deterministic --output-dir "$OUT/run" --json "$OUT/results.json" "${args[@]}" \ + >"$RUNNER_TEMP/console.log" 2>&1 || status=$? + echo "claude plugin eval exited $status" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The summary (health lines from the kept traces) and the validity + # verdict, read here because the traces stay on this runner. Only the + # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's + # detail lines can quote a trace. A leg that handed off reads nothing. + - name: Read the pass + id: read + if: steps.judge.outputs.handoff == 'false' + env: + OUT: ${{ runner.temp }}/pass + PLUGIN: ${{ inputs.plugin }} + PASS: ${{ inputs.pass }} + STAGE: ${{ runner.temp }}/upload + run: | + mkdir -p "$STAGE" + if [ ! -f "$OUT/results.json" ]; then + echo "::error::pass $PASS wrote no result" + exit 1 + fi + cp "$OUT/results.json" "$STAGE/results.json" + node .base/.github/scripts/plugin-eval/summarize.mjs "$OUT/results.json" --plugin "$PLUGIN" >"$STAGE/summary.md" || true + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$OUT/results.json" --runs 3 --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$(printf '%s\n' "$validity" | tail -n 1)" >"$STAGE/validity.txt" + cat "$STAGE/validity.txt" + sums=$(cd "$STAGE" && find . -type f -print0 | sort -z | xargs -0 sha256sum) + printf '%s\n' "$sums" >"$STAGE/manifest.sha256" + sum=$(sha256sum <"$STAGE/manifest.sha256") + echo "manifest-sha256=${sum%% *}" >>"$GITHUB_OUTPUT" + + - name: Scan the upload for credentials + id: scan-upload + if: steps.read.outcome == 'success' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/upload/ + + - name: Upload the pass + id: upload + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} + path: ${{ runner.temp }}/upload/ + retention-days: 1 + if-no-files-found: error + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: FALLBACK_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.pass-subscription.outputs.handoff-reason || 'no-key' }} diff --git a/.github/workflows/maintenance-measure-skill-edit.yml b/.github/workflows/maintenance-measure-skill-edit.yml index 6adf500dfc8..f65a5b5fea0 100644 --- a/.github/workflows/maintenance-measure-skill-edit.yml +++ b/.github/workflows/maintenance-measure-skill-edit.yml @@ -10,9 +10,10 @@ name: maintenance-measure-skill-edit # keeps a case once judged graders are dropped. # pass-a, the unchanged skill twice (eval-hill-climb-pass.yml), each on its # pass-a2 own runner, 3 runs per case per arm under --max-cost-usd 5. -# propose win-test.py's baseline gates, then one run-claude-cli leg -# (--restricted, read-only GITHUB_TOKEN, no write token, read-only -# tools) that returns one edit as structured output. +# propose win-test.py's baseline gates, then run-claude-cli on the key, +# subscription and fallback legs in turn (--restricted, read-only +# GITHUB_TOKEN, no write token, read-only tools), the first that +# finishes returning one edit as structured output. # scan no model, no secret: applies the edit to a fresh checkout and # refuses it outside plugins/

/skills//, under evals/, or # as select-plugins.mjs refuses a pull request's change. @@ -39,8 +40,10 @@ name: maintenance-measure-skill-edit # publish hold id-token, each to mint one lanes App token from the broker, # which refuses this caller on any event but workflow_dispatch and any actor # or re-runner but a listed human. No job holds a merge right, checks or -# statuses write, or workflows. The credential is the maintenance role's -# subscription token, as in eval-browser-dispatch.yml. How to stop it: set the +# statuses write, or workflows. Every pass and the proposal run on the +# maintenance role's chain, as in eval-browser-dispatch.yml: its API key, then +# its subscription token, then the central fallback token, one per job or +# step, each after the one before hands off. How to stop it: set the # organization or repository lanes kill-switch variable to anything but # "false"; every job that spends or writes checks it first, and the broker # reads it live. @@ -212,6 +215,7 @@ jobs: if: needs.gate.outputs.go == 'true' permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -220,13 +224,16 @@ jobs: pass: a base-sha: ${{ needs.gate.outputs.base-sha }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} pass-a2: needs: gate if: needs.gate.outputs.go == 'true' permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -235,14 +242,58 @@ jobs: pass: a2 base-sha: ${{ needs.gate.outputs.base-sha }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + + # The proposer's key-credit gate (ci-workflows check-key-credit, gate.cjs: + # one artifact-list call for the marker propose uploads after a key leg's + # credit stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour + # unset) starts the proposal on the subscription, with no refused key + # attempt first. No secret, no model, no checkout; every failure runs the + # key (skip-key empty). Probing is off. + propose-key-credit: + needs: [gate, pass-a, pass-a2] + if: needs.pass-a.result == 'success' && needs.pass-a2.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: propose + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: maintenance-measure-skill-edit + lane-id: maintenance-measure-skill-edit + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" - # The model, with the subscription token, a read-only GITHUB_TOKEN and + # The model, with one maintenance credential per leg (the API key, then the + # subscription token, then the fallback token), a read-only GITHUB_TOKEN and # nothing else. run-claude-cli runs the CLI with --restricted, which confines - # the file tools to the workspace; the leg gets Read, Grep and Glob only, so + # the file tools to the workspace; each leg gets Read, Grep and Glob only, so # its one result is the structured output, which the scan job checks. propose: - needs: [gate, pass-a, pass-a2] + needs: [gate, pass-a, pass-a2, propose-key-credit] if: needs.pass-a.result == 'success' && needs.pass-a2.result == 'success' runs-on: ubuntu-24.04 timeout-minutes: 40 @@ -385,22 +436,44 @@ jobs: printf '%s\n' "$prompt" >"$PROMPT_FILE" printf '%s\n' '{"type":"object","properties":{"file":{"type":"string","minLength":1,"maxLength":200},"old_string":{"type":"string","minLength":1,"maxLength":20000},"new_string":{"type":"string","maxLength":20000},"rationale":{"type":"string","maxLength":4000},"findings":{"type":"array","maxItems":10,"items":{"type":"string","maxLength":500}}},"required":["file","old_string","new_string","rationale","findings"],"additionalProperties":false}' >"$SCHEMA_FILE" - # Which legs can run, never a token's value. The maintenance role holds - # no API key (#13586): the subscription leg runs first, and the central - # fallback token's leg after it hands off (a limit stop, a credit stop, - # or a 401 or 402) or when its token is empty. + # Which legs can run, never a secret's value. The key runs unless it is + # empty or propose-key-credit skipped it; key-reason says why the + # proposal starts on the subscription. + - name: Check the API key + id: key + if: steps.prompt.outcome == 'success' + env: + KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.propose-key-credit.outputs.skip-key }} + run: | + key=$KEY + reason="" + if [ "$SKIP_KEY" = true ]; then + key=false + reason=credit-stop + elif [ "$KEY" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; the subscription leg proposes" + reason=no-key + fi + { echo "key=$key"; echo "key-reason=$reason"; } >>"$GITHUB_OUTPUT" + + # The subscription leg runs after the key leg hands off (a credit stop, + # or a 401 or 402) or did not run, and the central fallback token's leg + # after the subscription leg hands off (a limit stop, a credit stop, or + # a 401 or 402) or finds its token empty. - name: Check the credentials id: credential if: steps.prompt.outcome == 'success' env: + KEY_USABLE: ${{ steps.key.outputs.key }} PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} run: | - if [ "$PRESENT" != true ]; then + if [ "$KEY_USABLE" != true ] && [ "$PRESENT" != true ]; then if [ "$FALLBACK_PRESENT" = true ]; then echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; the fallback leg proposes on FALLBACK_CLAUDE_CODE_OAUTH_TOKEN" else - echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are both empty; no leg runs" + echo "::notice::no maintenance credential and no FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is usable; no leg runs" fi fi { echo "present=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" @@ -415,9 +488,52 @@ jobs: with: install-subprocess-isolation: "true" + # The three legs hold one credential each and stay in one job: the model + # has Read, Grep and Glob only, no Bash (run-claude-cli's one-job rule). + - name: Propose the edit on the API key + id: run-key + if: steps.prompt.outcome == 'success' && steps.key.outputs.key == 'true' + continue-on-error: true + timeout-minutes: 30 + uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + env: + CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1" + with: + lane: maintenance-measure-skill-edit + role: maintenance + prompt-file: ${{ runner.temp }}/hill-climb-prompt.md + preamble: You run unattended as the eval hill-climb CI lane. No one can answer a question during this run, and you change nothing. + api-key: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} + model: ${{ steps.models.outputs.model }} + effort: ${{ steps.models.outputs.effort }} + max-turns: ${{ steps.models.outputs.max-turns }} + permission-mode: dontAsk + tools: | + Read + Grep + Glob + allowed-tools: | + Read(./hill-climb-data/**) + Read(./.base/plugins/**) + disallowed-tools: | + Bash + Edit + Write + NotebookEdit + WebFetch + WebSearch + Agent + Skill + Read(//proc/**) + mcp__* + json-schema-file: ${{ runner.temp }}/hill-climb-schema.json + leg-timeout-minutes: "25" + - name: Propose the edit id: run - if: steps.prompt.outcome == 'success' && steps.credential.outputs.present == 'true' + if: >- + steps.prompt.outcome == 'success' && steps.credential.outputs.present == 'true' + && (steps.run-key.outcome == 'skipped' || steps.run-key.outputs.handoff == 'true') continue-on-error: true timeout-minutes: 30 uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 @@ -459,7 +575,8 @@ jobs: id: run-fallback if: >- steps.prompt.outcome == 'success' && steps.credential.outputs.fallback-ready == 'true' - && (steps.run.outputs.handoff == 'true' || steps.credential.outputs.present != 'true') + && (steps.run.outputs.handoff == 'true' + || steps.credential.outputs.present != 'true' && (steps.run-key.outcome == 'skipped' || steps.run-key.outputs.handoff == 'true')) continue-on-error: true timeout-minutes: 30 uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 @@ -496,6 +613,15 @@ jobs: json-schema-file: ${{ runner.temp }}/hill-climb-schema.json leg-timeout-minutes: "25" + - name: Upload the key leg's sealed transcript + if: always() && steps.run-key.outputs.execution-file != '' + continue-on-error: true + uses: ./.base/.github/actions/upload-lane-transcript # zizmor: ignore[self-repository] + with: + execution-file: ${{ steps.run-key.outputs.execution-file }} + public-key: ${{ vars.LANE_TRANSCRIPT_PUBLIC_KEY }} + job-id: ${{ github.job }}-key + - name: Upload the sealed transcript if: always() && steps.run.outputs.execution-file != '' continue-on-error: true @@ -514,6 +640,58 @@ jobs: public-key: ${{ vars.LANE_TRANSCRIPT_PUBLIC_KEY }} job-id: ${{ github.job }}-fallback + # Read by the next runs' credit gate (propose-key-credit): the artifact + # its `marker` output names (check-key-credit gate.cjs markerName), + # uploaded only after the key leg's credit-stop handoff. The name is + # written out so its retention row claims it. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.propose-key-credit.outputs.marker == 'key-credit-stop-maintenance-measure-skill-edit.yml-propose' && steps.run-key.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-maintenance-measure-skill-edit.yml-propose + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The key leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs. + - name: Record the key leg + if: always() && steps.run-key.outcome != 'skipped' && steps.run-key.outcome != '' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.run-key.outcome }} + execution-file: ${{ steps.run-key.outputs.execution-file }} + log-file: ${{ steps.run-key.outputs.log-file }} + lane: maintenance-measure-skill-edit + lane-id: maintenance-measure-skill-edit + credential: ${{ steps.run-key.outputs.credential }} + handoff: ${{ steps.run-key.outputs.handoff }} + known-failure-class: ${{ steps.run-key.outputs.lane-class }} + max-turns: ${{ steps.models.outputs.max-turns }} + fallback-reason: "" + # The subscription leg's lane-outcome record: its credential, outcome, handoff, # fallback reason and durationMs. - name: Record the subscription leg @@ -530,7 +708,7 @@ jobs: handoff: ${{ steps.run.outputs.handoff }} known-failure-class: ${{ steps.run.outputs.lane-class }} max-turns: ${{ steps.models.outputs.max-turns }} - fallback-reason: "" + fallback-reason: ${{ steps.run-key.outputs.handoff-reason || steps.key.outputs.key-reason }} # The fallback leg's lane-outcome record: its credential, outcome, handoff, # fallback reason and durationMs. @@ -548,7 +726,7 @@ jobs: handoff: ${{ steps.run-fallback.outputs.handoff }} known-failure-class: ${{ steps.run-fallback.outputs.lane-class }} max-turns: ${{ steps.models.outputs.max-turns }} - fallback-reason: ${{ steps.run.outputs.handoff-reason || steps.credential.outputs.present != 'true' && 'no-key' || '' }} + fallback-reason: ${{ steps.run.outputs.handoff-reason || 'no-key' }} # Read from the launcher's file only after the leg's init message reports # dontAsk; the scan job validates it. @@ -557,10 +735,11 @@ jobs: if: >- steps.run-fallback.outcome == 'success' && steps.run-fallback.outputs.structured-output-file != '' || steps.run.outcome == 'success' && steps.run.outputs.structured-output-file != '' + || steps.run-key.outcome == 'success' && steps.run-key.outputs.structured-output-file != '' env: - EXECUTION_FILE: ${{ steps.run-fallback.outputs.execution-file || steps.run.outputs.execution-file }} + EXECUTION_FILE: ${{ steps.run-fallback.outputs.execution-file || steps.run.outputs.execution-file || steps.run-key.outputs.execution-file }} MODE: dontAsk - OUTPUT_FILE: ${{ steps.run-fallback.outputs.structured-output-file || steps.run.outputs.structured-output-file }} + OUTPUT_FILE: ${{ steps.run-fallback.outputs.structured-output-file || steps.run.outputs.structured-output-file || steps.run-key.outputs.structured-output-file }} run: | bash .base/.github/scripts/read-patch/check-permission-mode.sh mkdir -p "$RUNNER_TEMP/proposal" @@ -689,6 +868,7 @@ jobs: if: ${{ !cancelled() && needs.scan.outputs.artifact-id != '' }} permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -699,7 +879,9 @@ jobs: edit-artifact-id: ${{ needs.scan.outputs.artifact-id }} edit-sha256: ${{ needs.scan.outputs.sha256 }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} # No model, no secret. Names the gate that stopped the run, with its # numbers; on a win, keeps the edit and each pass's results for 3 days. From c6ecbbd18c003528373e01ea3d78a44f0607fcf0 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:01:21 -0500 Subject: [PATCH 2/4] test(plugin-eval): remove the eval-handoff CLI test's temp folder on exit Co-Authored-By: Claude Opus 5.5 --- .github/scripts/plugin-eval/eval-handoff.test.mjs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs index 82ffd090aa8..88bc54fd6dc 100644 --- a/.github/scripts/plugin-eval/eval-handoff.test.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -8,11 +8,11 @@ // checkout at that pin; their messages are the observed shapes ci-workflows' // classify.test.cjs records (CLI 2.1.302). import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; -import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; +import { tempDir } from "../../../lib/test-tmp.mjs"; import { erroredTraces, handoffReason, judgeLegStreams } from "./eval-handoff.mjs"; test("a key leg hands off on a credit stop or a 401 or 402, never a 403 or a limit stop", () => { @@ -108,7 +108,7 @@ test("the real classifier: a subscription leg's limit stop and credit stop hand test("the CLI reads the probe and the errored runs, and writes the leg's duration for the lane record", { skip }, async () => { const { spawnSync } = await import("node:child_process"); - const dir = mkdtempSync(join(tmpdir(), "eval-handoff-cli-")); + const dir = tempDir("eval-handoff-cli-"); mkdirSync(join(dir, "t")); writeFileSync(join(dir, "probe.jsonl"), jsonl(CLEAN)); writeFileSync(join(dir, "t/1.jsonl"), jsonl(LIMITED)); From ec5bbeac327919535cd5401e2e533e2029e0264c Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:32:04 -0500 Subject: [PATCH 3/4] test(plugin-eval): run the eval-handoff contract against the pinned classifier in CI The real-classifier cases skipped unless a local ci-workflows checkout was named, so a run-claude-cli pin bump that reshaped judgeLeg's return passed every test. The test now fetches lane.cjs, classify.cjs and their helper at the pin the eval workflows check out, checks every leg names that one pin, and asserts the fields eval-handoff reads: failureClass, resets and the last message's api_error_status. Co-Authored-By: Claude Opus 5.5 --- .../scripts/plugin-eval/eval-handoff.test.mjs | 49 +++++++++++++++---- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs index 88bc54fd6dc..c97127bd997 100644 --- a/.github/scripts/plugin-eval/eval-handoff.test.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -4,9 +4,9 @@ // follows, and never from the fallback leg. The rule cases take their expected // values from that description (ci-workflows .github/actions/run-claude-cli/ // action.yml at b8bdd7f, the `handoff` output). The stream cases run the real -// lane.cjs and classify.cjs when EVAL_HANDOFF_CI_WORKFLOWS names a ci-workflows -// checkout at that pin; their messages are the observed shapes ci-workflows' -// classify.test.cjs records (CLI 2.1.302). +// lane.cjs and classify.cjs, fetched at the pin the eval workflows check out; +// their messages are the observed shapes ci-workflows' classify.test.cjs +// records (CLI 2.1.302). import assert from "node:assert/strict"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; @@ -62,14 +62,45 @@ test("the first stream that hands off decides, else the first stop class is kept assert.deepEqual(judgeLegStreams([], key, launcher), { handoff: false, reason: "", laneClass: "" }); }); -const ROOT = process.env.EVAL_HANDOFF_CI_WORKFLOWS ?? ""; -const real = ROOT === "" ? null : (() => { +// The classifier the legs check out, fetched at the pin the workflows name, so +// a pin bump that reshapes judgeLeg's return (failureClass, resets, the last +// message's api_error_status) or isCreditStop fails here. The three paths are +// the legs' sparse checkout. +const CLASSIFIER_FILES = [".github/actions/run-claude-cli/lane.cjs", ".github/actions/report-lane-outcome/classify.cjs", ".github/scripts/resolve-npm-tool-version.cjs"]; +const workflowPins = ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"].flatMap((name) => [ + ...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/repository: melodic-software\/ci-workflows\n\s+ref: ([0-9a-f]{40})/g), +].map((m) => m[1])); +const PIN = workflowPins[0]; +const ROOT = tempDir("eval-handoff-ci-workflows-"); +for (const path of CLASSIFIER_FILES) { + const url = `https://raw.githubusercontent.com/melodic-software/ci-workflows/${PIN}/${path}`; + const response = await fetch(url, { signal: AbortSignal.timeout(15000) }); + assert.ok(response.ok, `${url} answered ${response.status}`); + mkdirSync(join(ROOT, path, ".."), { recursive: true }); + writeFileSync(join(ROOT, path), await response.text()); +} +const real = (() => { const require = createRequire(join(ROOT, ".github/actions/run-claude-cli/lane.cjs")); const { judgeLeg, linesToMessages } = require("./lane.cjs"); const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); return { judgeLeg, linesToMessages, isCreditStop }; })(); -const skip = real === null ? "set EVAL_HANDOFF_CI_WORKFLOWS to a ci-workflows checkout at the run-claude-cli pin" : false; + +test("every eval leg checks out the classifier at one pin", () => { + assert.equal(workflowPins.length, 6, "three legs in each eval workflow"); + assert.deepEqual([...new Set(workflowPins)], [PIN]); +}); + +test("the pinned judgeLeg returns the fields eval-handoff reads", () => { + const dir = tempDir("eval-handoff-shape-"); + const executionFile = join(dir, "execution.json"); + writeFileSync(executionFile, JSON.stringify(real.linesToMessages(jsonl(LIMITED)))); + const judged = real.judgeLeg({ leg: { leg: "subscription" }, executionFile, exitCode: 1, timedOut: false, maxTurns: "" }); + assert.equal(judged.failureClass, "rate-limit"); + assert.equal(judged.resets, "2100-01-01T00:00:00Z"); + assert.equal(judged.messages.at(-1).api_error_status, 429); + assert.equal(real.isCreditStop(SPEND_LIMIT, { exitCode: 1 }), true); +}); const jsonl = (messages) => messages.map((m) => JSON.stringify(m)).join("\n"); const SPEND_LIMIT_TEXT = "API Error: 400 You have reached your specified workspace API usage limits. You will regain access on 2026-11-01 at 00:00 UTC."; @@ -90,7 +121,7 @@ const LIMITED = [ ]; const CLEAN = [{ type: "system", subtype: "init" }, { type: "result", subtype: "success", is_error: false, num_turns: 3 }]; -test("the real classifier: a key leg's observed spend-limit stop and 401 hand off, a 403 does not", { skip }, () => { +test("the real classifier: a key leg's observed spend-limit stop and 401 hand off, a 403 does not", () => { const key = { leg: "api-key", fallbackReady: false }; assert.deepEqual(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], key, real), { handoff: true, reason: "credit-stop", laneClass: "credit-stop" }); assert.deepEqual(judgeLegStreams([{ text: jsonl(rejected(401)), exitCode: 1 }], key, real), { handoff: true, reason: "auth", laneClass: "auth" }); @@ -98,7 +129,7 @@ test("the real classifier: a key leg's observed spend-limit stop and 401 hand of assert.deepEqual(judgeLegStreams([{ text: jsonl(CLEAN), exitCode: 0 }], key, real), { handoff: false, reason: "", laneClass: "" }); }); -test("the real classifier: a subscription leg's limit stop and credit stop hand off to the fallback leg", { skip }, () => { +test("the real classifier: a subscription leg's limit stop and credit stop hand off to the fallback leg", () => { const sub = { leg: "subscription", fallbackReady: true }; assert.deepEqual(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], sub, real), { handoff: true, reason: "limit-stop", laneClass: "rate-limit" }); assert.equal(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], sub, real).reason, "credit-stop"); @@ -106,7 +137,7 @@ test("the real classifier: a subscription leg's limit stop and credit stop hand assert.equal(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], { leg: "fallback", fallbackReady: true }, real).handoff, false); }); -test("the CLI reads the probe and the errored runs, and writes the leg's duration for the lane record", { skip }, async () => { +test("the CLI reads the probe and the errored runs, and writes the leg's duration for the lane record", async () => { const { spawnSync } = await import("node:child_process"); const dir = tempDir("eval-handoff-cli-"); mkdirSync(join(dir, "t")); From ae1e5ce91361ae4fd7ec205372934ffb2db92735 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 18:53:29 -0500 Subject: [PATCH 4/4] test(lanes): allow the fallback token only in a plugin eval leg's eval and probe steps The claude-cli clause of readsFallback accepted the fallback token in any `claude ` run step of any lane. It now accepts it only when every step that reads it is a `claude plugin eval` run or that job's `probe` step, each as CLAUDE_CODE_OAUTH_TOKEN, and a fixture shows any other step on the token failing. Co-Authored-By: Claude Opus 5.5 --- .../actions/resolve-config/callers.test.mjs | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 6f0e60b5fa2..830d3a288e1 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -2147,20 +2147,47 @@ test("every job that reads a Claude secret is a catalog lane, and every catalog // (intake-triage.yml and pr-review-cli.yml from v0.65.12, ci-workflows#890), // or a job that runs the fallback leg itself, passing it to run-claude-cli as // fallback-token, to the read runner as its third call's -// fallback-claude-code-oauth-token, or to a `claude plugin eval` leg's claude -// steps as CLAUDE_CODE_OAUTH_TOKEN; or a call of the eval pass workflow, which -// runs that leg. Any job may ask only whether it is set. +// fallback-claude-code-oauth-token, or to a `claude plugin eval` leg's eval +// and probe steps, and no other step, as CLAUDE_CODE_OAUTH_TOKEN; or a call of +// the eval pass workflow, which runs that leg. Any job may ask only whether it +// is set. const FALLBACK_SECRET = "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"; const FALLBACK_REUSABLE = /^melodic-software\/ci-workflows\/\.github\/workflows\/(?:intake-triage|pr-review-cli)\.yml@main$/; const EVAL_PASS_REUSABLE = "./.github/workflows/eval-hill-climb-pass.yml"; const FALLBACK_VALUE = `\${{ secrets.${FALLBACK_SECRET} }}`; +// A plugin eval leg's claude steps: the `claude plugin eval` run, and the +// policy probe (step id `probe`, a `claude -p`) in a job that runs one. +const EVAL_RUN = /^\s*claude plugin eval /m; +const isEvalLegStep = (step, job) => + isLauncherStep(step, "claude-cli") && + (EVAL_RUN.test(step.run) || (step.id === "probe" && /^\s*claude -p /m.test(step.run) && (job.steps ?? []).some((s) => EVAL_RUN.test(String(s.run ?? ""))))); +// The fallback token reaches only a plugin eval leg's claude steps, as their +// CLAUDE_CODE_OAUTH_TOKEN; any other step that reads it disqualifies the job. +const evalLegReadsFallback = (job) => { + const readers = (job.steps ?? []).filter((step) => claudeReads(withoutPresenceChecks(step)).includes(FALLBACK_SECRET)); + return readers.length > 0 && readers.every((step) => isEvalLegStep(step, job) && step.env?.CLAUDE_CODE_OAUTH_TOKEN === FALLBACK_VALUE); +}; const readsFallback = (job) => (FALLBACK_REUSABLE.test(String(job.uses ?? "")) && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || (job.uses === EVAL_PASS_REUSABLE && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || (job.steps ?? []).some((step) => isLauncherStep(step, "run-claude-cli") && step.with?.["fallback-token"] === FALLBACK_VALUE) || - (job.steps ?? []).some((step) => isLauncherStep(step, "claude-cli") && step.env?.CLAUDE_CODE_OAUTH_TOKEN === FALLBACK_VALUE) || + evalLegReadsFallback(job) || (job.uses === READ_RUNNER && job.with?.fallback === "fallback" && job.secrets?.["fallback-claude-code-oauth-token"] === FALLBACK_VALUE) || (claudeReads(job).includes(FALLBACK_SECRET) && !claudeReads(withoutPresenceChecks(job)).includes(FALLBACK_SECRET)); +test("the fallback token is allowed only in a plugin eval leg's eval and probe steps", () => { + const token = { CLAUDE_CODE_OAUTH_TOKEN: FALLBACK_VALUE }; + const evalStep = { name: "Run the pass", env: token, run: "set -e\nclaude plugin eval x --json r.json" }; + const probe = { name: "Probe", id: "probe", env: token, run: 'claude -p "probe" --output-format stream-json' }; + const other = { name: "Ask", env: token, run: 'claude -p "summarize the diff"' }; + const presence = { name: "Check", env: { P: `\${{ secrets.${FALLBACK_SECRET} != '' }}` }, run: "true" }; + assert.equal(readsFallback({ steps: [presence, probe, evalStep] }), true, "an eval leg"); + assert.equal(readsFallback({ steps: [evalStep] }), true, "an eval leg with no probe"); + assert.equal(readsFallback({ steps: [other] }), false, "a claude -p step that runs no eval"); + assert.equal(readsFallback({ steps: [probe] }), false, "a probe in a job that runs no eval"); + assert.equal(readsFallback({ steps: [probe, evalStep, other] }), false, "an eval leg with another step on the token"); + assert.equal(readsFallback({ steps: [{ ...evalStep, env: { ANY_NAME: FALLBACK_VALUE } }] }), false, "the token under another name"); +}); + test("every catalog lane reads only its role's secrets, and every role: it passes is its role", () => { const jobs = claudeSecretJobs(); for (const { role, lane, key } of catalogLanes()) {