diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index fd6569fefe1..830d3a288e1 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -2146,16 +2146,48 @@ test("every job that reads a Claude secret is a catalog lane, and every catalog // which only a call of a ci-workflows reusable with a fallback job may pass // (intake-triage.yml and pr-review-cli.yml from v0.65.12, ci-workflows#890), // or a job that runs the fallback leg itself, passing it to run-claude-cli as -// fallback-token or to the read runner as its third call's -// fallback-claude-code-oauth-token. Any job may ask only whether it is set. +// fallback-token, to the read runner as its third call's +// fallback-claude-code-oauth-token, or to a `claude plugin eval` leg's eval +// and probe steps, and no other step, as CLAUDE_CODE_OAUTH_TOKEN; or a call of +// the eval pass workflow, which runs that leg. Any job may ask only whether it +// is set. const FALLBACK_SECRET = "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"; const FALLBACK_REUSABLE = /^melodic-software\/ci-workflows\/\.github\/workflows\/(?:intake-triage|pr-review-cli)\.yml@main$/; +const EVAL_PASS_REUSABLE = "./.github/workflows/eval-hill-climb-pass.yml"; const FALLBACK_VALUE = `\${{ secrets.${FALLBACK_SECRET} }}`; +// A plugin eval leg's claude steps: the `claude plugin eval` run, and the +// policy probe (step id `probe`, a `claude -p`) in a job that runs one. +const EVAL_RUN = /^\s*claude plugin eval /m; +const isEvalLegStep = (step, job) => + isLauncherStep(step, "claude-cli") && + (EVAL_RUN.test(step.run) || (step.id === "probe" && /^\s*claude -p /m.test(step.run) && (job.steps ?? []).some((s) => EVAL_RUN.test(String(s.run ?? ""))))); +// The fallback token reaches only a plugin eval leg's claude steps, as their +// CLAUDE_CODE_OAUTH_TOKEN; any other step that reads it disqualifies the job. +const evalLegReadsFallback = (job) => { + const readers = (job.steps ?? []).filter((step) => claudeReads(withoutPresenceChecks(step)).includes(FALLBACK_SECRET)); + return readers.length > 0 && readers.every((step) => isEvalLegStep(step, job) && step.env?.CLAUDE_CODE_OAUTH_TOKEN === FALLBACK_VALUE); +}; const readsFallback = (job) => (FALLBACK_REUSABLE.test(String(job.uses ?? "")) && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || + (job.uses === EVAL_PASS_REUSABLE && job.secrets?.[FALLBACK_SECRET] === FALLBACK_VALUE) || (job.steps ?? []).some((step) => isLauncherStep(step, "run-claude-cli") && step.with?.["fallback-token"] === FALLBACK_VALUE) || + evalLegReadsFallback(job) || (job.uses === READ_RUNNER && job.with?.fallback === "fallback" && job.secrets?.["fallback-claude-code-oauth-token"] === FALLBACK_VALUE) || (claudeReads(job).includes(FALLBACK_SECRET) && !claudeReads(withoutPresenceChecks(job)).includes(FALLBACK_SECRET)); +test("the fallback token is allowed only in a plugin eval leg's eval and probe steps", () => { + const token = { CLAUDE_CODE_OAUTH_TOKEN: FALLBACK_VALUE }; + const evalStep = { name: "Run the pass", env: token, run: "set -e\nclaude plugin eval x --json r.json" }; + const probe = { name: "Probe", id: "probe", env: token, run: 'claude -p "probe" --output-format stream-json' }; + const other = { name: "Ask", env: token, run: 'claude -p "summarize the diff"' }; + const presence = { name: "Check", env: { P: `\${{ secrets.${FALLBACK_SECRET} != '' }}` }, run: "true" }; + assert.equal(readsFallback({ steps: [presence, probe, evalStep] }), true, "an eval leg"); + assert.equal(readsFallback({ steps: [evalStep] }), true, "an eval leg with no probe"); + assert.equal(readsFallback({ steps: [other] }), false, "a claude -p step that runs no eval"); + assert.equal(readsFallback({ steps: [probe] }), false, "a probe in a job that runs no eval"); + assert.equal(readsFallback({ steps: [probe, evalStep, other] }), false, "an eval leg with another step on the token"); + assert.equal(readsFallback({ steps: [{ ...evalStep, env: { ANY_NAME: FALLBACK_VALUE } }] }), false, "the token under another name"); +}); + test("every catalog lane reads only its role's secrets, and every role: it passes is its role", () => { const jobs = claudeSecretJobs(); for (const { role, lane, key } of catalogLanes()) { @@ -2179,11 +2211,23 @@ test("every catalog lane reads only its role's secrets, and every role: it passe } }); -test("a job reads a Claude API key only when its catalog lane runs on run-claude-cli", () => { +// The maintenance role's `claude plugin eval` lanes take its key too (owner +// decision Q2 on #13586): plugin eval runs on the credential in the +// environment, ANTHROPIC_API_KEY for a Console key +// (code.claude.com/docs/en/plugin-evals, "Run evals in CI"). +test("a job reads a Claude API key only when its catalog lane runs on run-claude-cli or is a plugin eval leg", () => { const lanes = new Map(catalogLanes().map(({ lane, key }) => [key, lane])); const keyed = [...claudeSecretJobs()].filter(([, { names }]) => names.some(isApiKey)); assert.ok(keyed.length > 0); - for (const [key] of keyed) assert.equal(lanes.get(key)?.launcher, "run-claude-cli", key); + for (const [key, { job }] of keyed) { + const launcher = lanes.get(key)?.launcher; + if (launcher === "claude-cli") { + const evalRun = (job.steps ?? []).some((step) => isLauncherStep(step, "claude-cli") && /^\s*claude plugin eval /m.test(step.run)); + assert.ok(evalRun || job.uses === EVAL_PASS_REUSABLE, `${key} reads a key but runs no claude plugin eval`); + } else { + assert.equal(launcher, "run-claude-cli", key); + } + } }); test("a Claude credential reaches only the launcher step of its catalog lane", () => { @@ -9047,15 +9091,121 @@ test("maintenance-tidy-code-upkeep.yml: the sweep runs key, subscription and fal for (const id of ["model-subscription", "model-fallback"]) assert.doesNotMatch(JSON.stringify(jobs[id]), /key-credit-stop/, id); }); +// The maintenance role's `claude plugin eval` lanes run key, subscription and +// fallback legs, one credential per job, each the key leg's steps but for the +// credential, the leg its handoff judge names, the key's credit-stop marker +// and the leg record's credential and fallback reason. A key reaches the CLI +// as ANTHROPIC_API_KEY only inside the claude step, LANE_API_KEY unset first, +// so CLAUDE_CODE_SUBPROCESS_ENV_SCRUB strips it from Bash subprocesses. +const EVAL_LEG_CREDENTIALS = { + key: ["LANE_API_KEY", "${{ secrets.MAINTENANCE_CLAUDE_API_KEY }}"], + subscription: ["CLAUDE_CODE_OAUTH_TOKEN", "${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }}"], + fallback: ["CLAUDE_CODE_OAUTH_TOKEN", "${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }}"], +}; +const CLASSIFIER_PIN = "b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6"; +for (const [file, gateId, [keyId, subId, fbId], marker] of [ + ["eval-hill-climb-pass.yml", "gate", ["pass", "pass-subscription", "pass-fallback"], "key-credit-stop-maintenance-measure-skill-edit.yml-pass"], + ["eval-browser-dispatch.yml", "eval-key-credit", ["eval", "eval-subscription", "eval-fallback"], "key-credit-stop-eval-browser-dispatch.yml-eval"], +]) { + test(`${file}: the eval runs key, subscription and fallback legs, each with one credential`, () => { + const { jobs } = workflow(file); + const legs = { key: keyId, subscription: subId, fallback: fbId }; + for (const [leg, id] of Object.entries(legs)) { + const steps = jobs[id].steps; + const launchers = steps.filter((s) => isLauncherStep(s, "claude-cli")); + assert.ok(launchers.some((s) => /^\s*claude plugin eval /m.test(s.run)), `${id} runs the eval`); + const [name, value] = EVAL_LEG_CREDENTIALS[leg]; + for (const step of launchers) { + assert.equal(step.env[name], value, `${id} ${step.name}`); + assert.deepEqual(stepSecrets(step), [value.match(/secrets\.(\w+)/)[1]], `${id} ${step.name} holds one credential`); + const firstClaude = step.run.search(/^\s*claude\s/m); + assert.ok(step.run.indexOf("unset LANE_API_KEY") < firstClaude, `${id} ${step.name} unsets LANE_API_KEY before claude runs`); + assert.match(step.env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB, /^(?:"?1"?|\$\{\{ .+ \}\})$/, `${id} ${step.name} scrubs Bash subprocesses`); + } + assert.equal(steps.find((s) => s.id === "judge").env.LEG, leg, id); + const checkout = steps.find((s) => s.with?.repository === "melodic-software/ci-workflows"); + assert.equal(checkout.with.ref, CLASSIFIER_PIN, `${id} reads the classifier at the run-claude-cli pin`); + assert.equal(checkout.with["persist-credentials"], false); + assert.ok(!jobs[id].permissions || !Object.values(jobs[id].permissions).includes("write"), `${id} holds no write grant`); + const record = steps.find((s) => s.name === "Record the leg"); + assert.equal(record.with.credential, value.match(/secrets\.(\w+)/)[1], id); + assert.equal(record.with["execution-file"], "${{ steps.judge.outputs.leg-result }}", id); + } + // The run-claude-cli pin the classifier checkout follows is the one this repository runs. + assert.match(read("../../workflows/maintenance-measure-skill-edit.yml"), new RegExp(`run-claude-cli@${CLASSIFIER_PIN} `)); + + const shape = (id) => + jobs[id].steps + .filter((s) => !["credential", "credit-marker", "credit-marker-scan", "credit-marker-upload"].includes(s.id)) + .map((s) => { + const copy = structuredClone(s); + for (const k of ["LANE_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", "LEG"]) delete copy.env?.[k]; + if (copy.name === "Record the leg") for (const k of ["credential", "fallback-reason"]) delete copy.with[k]; + return copy; + }); + assert.deepEqual(shape(subId), shape(keyId)); + assert.deepEqual(shape(fbId), shape(keyId)); + assert.deepEqual(Object.keys(jobs[subId].outputs), Object.keys(jobs[keyId].outputs)); + + const ctx = (over) => ({ + [`needs.${gateId}.result`]: "success", + [`needs.${gateId}.outputs.skip-key`]: "", + "github.event_name": "workflow_dispatch", + "github.ref": "refs/heads/main", + "github.event.repository.default_branch": "main", + "github.event.label.name": "", + "github.event.pull_request.head.repo.full_name": "", + "github.repository": "melodic-software/claude-code-plugins", + "github.event.pull_request.draft": false, + ...over, + }); + const keyIf = jobs[keyId].if.replaceAll("format('refs/heads/{0}', github.event.repository.default_branch)", "'refs/heads/main'"); + assert.equal(evaluate(keyIf, ctx({})), true); + assert.equal(evaluate(keyIf, ctx({ [`needs.${gateId}.result`]: "failure" })), false); + const sub = (over) => + evaluate(jobs[subId].if, ctx({ [`needs.${keyId}.outputs.handoff`]: "", [`needs.${keyId}.outputs.credential-present`]: "true", ...over })); + assert.equal(sub({ [`needs.${keyId}.outputs.handoff`]: "true" }), true, "a handed-off key"); + assert.equal(sub({ [`needs.${keyId}.outputs.credential-present`]: "false" }), true, "an empty key"); + assert.equal(sub({ [`needs.${gateId}.outputs.skip-key`]: "true" }), true, "a credit-stopped key"); + assert.equal(sub({ [`needs.${keyId}.outputs.handoff`]: "false" }), false, "a key leg that finished"); + assert.equal(sub({}), false, "a key leg that failed before its judge"); + const fb = (handoff, present, ready) => + evaluate(jobs[fbId].if, { + [`needs.${subId}.outputs.handoff`]: handoff, + [`needs.${subId}.outputs.credential-present`]: present, + [`needs.${subId}.outputs.fallback-ready`]: ready, + }); + assert.equal(fb("true", "true", "true"), true); + assert.equal(fb("", "false", "true"), true, "an empty subscription token hands to the fallback leg"); + assert.equal(fb("", "false", "false"), false, "the fallback leg starts only when its token is set"); + assert.equal(fb("true", "true", "false"), false); + assert.equal(fb("false", "true", "true"), false); + assert.equal( + jobs[subId].steps.find((s) => s.name === "Record the leg").with["fallback-reason"], + `\${{ needs.${keyId}.outputs.handoff-reason || needs.${gateId}.outputs.skip-key == 'true' && 'credit-stop' || needs.${keyId}.outputs.credential-present == 'false' && 'no-key' || '' }}`, + ); + assert.equal(jobs[fbId].steps.find((s) => s.name === "Record the leg").with["fallback-reason"], `\${{ needs.${subId}.outputs.handoff-reason || 'no-key' }}`); + + // The gate reads markers with actions: read and nothing else; only the key leg uploads one. + assert.deepEqual(jobs[gateId].permissions, { actions: "read" }); + assert.equal(claudeReads(withoutPresenceChecks(jobs[gateId])).length, 0, "the gate reads no Claude credential"); + const credit = jobs[gateId].steps.find((s) => s.id === "credit"); + assert.equal(credit.with["key-job"], keyId); + assert.equal(credit.with["probe-minutes"], "0"); + const upload = jobs[keyId].steps.find((s) => s.id === "credit-marker-upload"); + assert.equal(upload.with.name, marker); + assert.equal(upload["continue-on-error"], true); + assert.match(jobs[keyId].steps.find((s) => s.id === "credit-marker").if, new RegExp(`'${marker.replaceAll(".", "\\.")}' && steps\\.judge\\.outputs\\.handoff-reason == 'credit-stop'`)); + for (const id of [subId, fbId]) assert.doesNotMatch(JSON.stringify(jobs[id]), /key-credit-stop/, id); + assert.equal(JSON.parse(read("../../artifact-retention.json")).families[marker]?.["retention-days"], 1, `${marker} has a retention row`); + }); +} + // Every job that runs a run-claude-cli leg writes that leg's lane-outcome // record (report-lane-outcome with a lane-id): the record carries the leg's // credential, outcome, handoff and durationMs, which lane telemetry // aggregates per leg (#13586). -const RECORDLESS_LEGS = new Set([ - // The maintenance role's proposal pass, open on #13586 with that role's - // missing key; its lane record comes with its legs. - "maintenance-measure-skill-edit.yml propose", -]); +const RECORDLESS_LEGS = new Set([]); test("every run-claude-cli job writes a lane record for its leg", () => { const missing = []; for (const file of workflowFiles()) { @@ -9196,18 +9346,11 @@ function missingLegs(catalog, launched) { } return missing; } -const EVAL_REASON = "the maintenance role holds no API key (keyed false, no MAINTENANCE_CLAUDE_API_KEY), and claude plugin eval runs outside run-claude-cli, so it has no handoff to a fallback leg (#13586)"; // Lanes that cannot take every leg yet (#13586): each names the exact legs it // waives and why. The test fails when an exempt lane lacks a leg its entry // does not name, or runs a leg its entry waives, so an entry never covers // more than it says and goes once its lane gains the leg. const LEG_EXEMPTIONS = { - "eval-browser-dispatch.yml maintenance eval": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "eval-hill-climb-pass.yml maintenance pass": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-a": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-a2": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance pass-b": { legs: ["key", "fallback"], reason: EVAL_REASON }, - "maintenance-measure-skill-edit.yml maintenance propose": { legs: ["key"], reason: "the maintenance role holds no API key (keyed false, no MAINTENANCE_CLAUDE_API_KEY) (#13586)" }, "post-merge-test-mutation.yml automations audit": { legs: ["key", "fallback"], reason: "it runs on claude-code-action, which takes no API key here (only run-claude-cli does) and has no handoff to a fallback leg; both wait on the audit's move to run-claude-cli (#13586)", diff --git a/.github/artifact-retention.json b/.github/artifact-retention.json index 9bea06e0a43..1f873d1f4ba 100644 --- a/.github/artifact-retention.json +++ b/.github/artifact-retention.json @@ -77,6 +77,30 @@ "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", "scan": "upload" }, + "key-credit-stop-eval-browser-dispatch.yml-eval": { + "retention-days": 1, + "readers": [ + ".github/workflows/eval-browser-dispatch.yml eval-key-credit's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention)" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, + "key-credit-stop-maintenance-measure-skill-edit.yml-pass": { + "retention-days": 1, + "readers": [ + ".github/workflows/eval-hill-climb-pass.yml gate's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention); the name carries maintenance-measure-skill-edit.yml because a called workflow's github context is its caller's" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, + "key-credit-stop-maintenance-measure-skill-edit.yml-propose": { + "retention-days": 1, + "readers": [ + ".github/workflows/maintenance-measure-skill-edit.yml propose-key-credit's \"Read the key's credit history\" step runs ci-workflows check-key-credit, which reads the newest marker of this exact name inside KEY_CREDIT_WINDOW_HOURS (1 hour unset, 24 at most for this retention)" + ], + "reason": "The marker only says the key hit a credit stop; the gate counts it for its window, an hour by default, so a day covers any window up to 24 hours and an expired marker runs the key, as before the gate.", + "scan": "upload" + }, "key-credit-stop-pr-verify-capture.yml-capture": { "retention-days": 1, "readers": [ diff --git a/.github/lane-funding.json b/.github/lane-funding.json index f4e7f76eda6..479047ff177 100644 --- a/.github/lane-funding.json +++ b/.github/lane-funding.json @@ -127,12 +127,16 @@ "keyed": true, "category": "maintenance", "lanes": [ - { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-a", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-a2", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "propose", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "run-claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass-b", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }, - { "id": "pass", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] } + { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY"] }, + { "id": "eval-subscription", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "eval-fallback", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-a", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-a2", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "propose", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "run-claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-b", "workflow": "maintenance-measure-skill-edit.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY", "MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_API_KEY"] }, + { "id": "pass-subscription", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN", "FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] }, + { "id": "pass-fallback", "workflow": "eval-hill-climb-pass.yml", "launcher": "claude-cli", "triggers": ["workflow_call"], "currentSecrets": ["FALLBACK_CLAUDE_CODE_OAUTH_TOKEN"] } ] } ] diff --git a/.github/scripts/plugin-eval/eval-handoff.mjs b/.github/scripts/plugin-eval/eval-handoff.mjs new file mode 100755 index 00000000000..4187905dd96 --- /dev/null +++ b/.github/scripts/plugin-eval/eval-handoff.mjs @@ -0,0 +1,155 @@ +#!/usr/bin/env node +// eval-handoff.mjs: whether one `claude plugin eval` leg hands off to the next +// credential, judged by run-claude-cli's own classifier. +// +// A leg is the policy probe (`claude -p --output-format stream-json`) plus the +// eval it guards, both on one credential. Each kept run trace is the session +// as stream-json, one message per line (code.claude.com/docs/en/plugin-evals, +// "What a grader can look at", the `trace` row), the same stream run-claude-cli +// writes as its execution file. So every errored run's trace and the probe's +// stream go through lane.cjs judgeLeg and classify.cjs isCreditStop from the +// ci-workflows checkout at the pin the lanes run (CI_WORKFLOWS), and the leg +// hands off on the rule lane.cjs handoffReason applies, restated below because +// lane.cjs does not export it: +// key leg a credit stop, or class auth with API status 401 or 402 +// subscription leg with FALLBACK_READY true: a credit stop, a 401 or 402, +// or a limit stop (a rejected rate_limit_event with its reset) +// fallback leg never +// A run whose `error` is null ended normally and is not judged; the suite's +// exit code is not read, since a case below --threshold also exits 1. +// +// Env: LEG (key, subscription or fallback), FALLBACK_READY, CI_WORKFLOWS, +// RESULTS (the --json results, one path per line; a missing one is skipped), +// PROBE and PROBE_EXIT (the probe's stream and exit code; PROBE may be empty), +// LEG_RESULT (where to write the leg's execution file for report-lane-outcome: +// one result message carrying the suites' summed durationSeconds and costUsd, +// nothing a run wrote). +// Prints handoff, handoff-reason, lane-class and outcome as key=value lines. +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join } from "node:path"; +import process from "node:process"; +import { fileURLToPath } from "node:url"; + +const LEGS = { key: "api-key", subscription: "subscription", fallback: "fallback" }; +const REFUSED_STATUSES = [401, 402]; +// The classes a run's trace can show that say the credential stopped, not the case. +const STOP_CLASSES = ["credit-stop", "auth", "rate-limit", "overloaded"]; + +export function handoffReason(leg, failureClass, status, { fallbackReady = false, limitStop = false, creditStop = false } = {}) { + const refused = failureClass === "auth" && REFUSED_STATUSES.includes(status); + if (leg === "api-key") { + if (failureClass === "credit-stop") return "credit-stop"; + return refused ? "auth" : ""; + } + if (leg !== "subscription" || !fallbackReady || failureClass === "") return ""; + if (creditStop) return "credit-stop"; + if (refused) return "auth"; + return failureClass === "rate-limit" && limitStop ? "limit-stop" : ""; +} + +// Every errored run's trace path, resolved as run-validity.py resolves it. +export function erroredTraces(result, base) { + const traces = []; + for (const kase of Array.isArray(result?.cases) ? result.cases : []) { + for (const runs of Object.values(kase?.arms ?? {})) { + for (const run of Array.isArray(runs) ? runs : []) { + if (run?.error === null || run?.error === undefined) continue; + const path = run.tracePath; + if (typeof path === "string" && path !== "") traces.push(isAbsolute(path) ? path : join(base, path)); + } + } + } + return traces; +} + +// One stream's verdict: its class and why it hands off ("" when it does not). +export function judgeStream(text, exitCode, { leg, fallbackReady }, { judgeLeg, linesToMessages, isCreditStop }) { + const dir = mkdtempSync(join(tmpdir(), "eval-handoff-")); + try { + const executionFile = join(dir, "execution.json"); + writeFileSync(executionFile, JSON.stringify(linesToMessages(text))); + const judged = judgeLeg({ leg: { leg }, executionFile, exitCode, timedOut: false, maxTurns: "" }); + const status = judged.messages.at(-1)?.api_error_status; + const reason = handoffReason(leg, judged.failureClass, Number.isInteger(status) ? status : null, { + fallbackReady, + limitStop: judged.failureClass === "rate-limit" && judged.resets !== null, + creditStop: leg === "subscription" && isCreditStop(judged.messages, { exitCode }), + }); + return { failureClass: judged.failureClass, reason }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +// The leg's verdict over the probe and every errored run: the first stream +// that hands off decides; otherwise the first stop class any stream shows. +export function judgeLegStreams(streams, options, launcher) { + let laneClass = ""; + for (const { text, exitCode } of streams) { + const { failureClass, reason } = judgeStream(text, exitCode, options, launcher); + if (reason !== "") return { handoff: true, reason, laneClass: failureClass }; + if (laneClass === "" && STOP_CLASSES.includes(failureClass)) laneClass = failureClass; + } + return { handoff: false, reason: "", laneClass }; +} + +function readJson(path) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch { + return null; + } +} + +function main(env) { + const leg = LEGS[env.LEG]; + if (!leg) throw new Error(`LEG must be key, subscription or fallback, not ${JSON.stringify(env.LEG)}`); + const root = env.CI_WORKFLOWS ?? ""; + const require = createRequire(join(root, ".github/actions/run-claude-cli/lane.cjs")); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + const streams = []; + if (env.PROBE && existsSync(env.PROBE)) { + const exitCode = Number.parseInt(env.PROBE_EXIT ?? "", 10); + streams.push({ text: readFileSync(env.PROBE, "utf8"), exitCode: Number.isInteger(exitCode) ? exitCode : 1 }); + } + let seconds = null; + let cost = null; + const add = (sum, value) => (typeof value === "number" && value >= 0 ? (sum ?? 0) + value : sum); + for (const file of (env.RESULTS ?? "").split("\n").filter(Boolean)) { + const result = readJson(file); + seconds = add(seconds, result?.durationSeconds); + cost = add(cost, result?.costUsd); + for (const trace of erroredTraces(result, dirname(file))) { + if (existsSync(trace)) streams.push({ text: readFileSync(trace, "utf8"), exitCode: 1 }); + } + } + const verdict = judgeLegStreams(streams, { leg, fallbackReady: env.FALLBACK_READY === "true" }, { judgeLeg, linesToMessages, isCreditStop }); + if (env.LEG_RESULT) { + const message = { + type: "result", + subtype: verdict.laneClass === "" ? "success" : "error_during_execution", + is_error: verdict.laneClass !== "", + ...(seconds === null ? {} : { duration_ms: Math.round(seconds * 1000) }), + ...(cost === null ? {} : { total_cost_usd: cost }), + }; + writeFileSync(env.LEG_RESULT, `${JSON.stringify([message])}\n`); + } + return [ + `handoff=${verdict.handoff}`, + `handoff-reason=${verdict.reason}`, + `lane-class=${verdict.laneClass}`, + `outcome=${verdict.laneClass === "" ? "success" : "failure"}`, + ].join("\n"); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + try { + console.log(main(process.env)); + } catch (error) { + console.error(`eval-handoff: ${error.message}`); + process.exit(2); + } +} diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs new file mode 100644 index 00000000000..c97127bd997 --- /dev/null +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -0,0 +1,162 @@ +// eval-handoff.mjs hands an eval leg to the next credential on exactly the +// stops run-claude-cli's action.yml names: a key leg's credit stop or 401/402, +// a subscription leg's credit stop, 401/402 or limit stop when a fallback leg +// follows, and never from the fallback leg. The rule cases take their expected +// values from that description (ci-workflows .github/actions/run-claude-cli/ +// action.yml at b8bdd7f, the `handoff` output). The stream cases run the real +// lane.cjs and classify.cjs, fetched at the pin the eval workflows check out; +// their messages are the observed shapes ci-workflows' classify.test.cjs +// records (CLI 2.1.302). +import assert from "node:assert/strict"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { test } from "node:test"; +import { tempDir } from "../../../lib/test-tmp.mjs"; +import { erroredTraces, handoffReason, judgeLegStreams } from "./eval-handoff.mjs"; + +test("a key leg hands off on a credit stop or a 401 or 402, never a 403 or a limit stop", () => { + assert.equal(handoffReason("api-key", "credit-stop", 400), "credit-stop"); + assert.equal(handoffReason("api-key", "auth", 401), "auth"); + assert.equal(handoffReason("api-key", "auth", 402), "auth"); + assert.equal(handoffReason("api-key", "auth", 403), "", "a 403 stops the run"); + assert.equal(handoffReason("api-key", "auth", null), "", "auth with no status"); + assert.equal(handoffReason("api-key", "rate-limit", 429, { limitStop: true }), ""); + assert.equal(handoffReason("api-key", "", null), ""); +}); + +test("a subscription leg hands off only when a fallback leg follows", () => { + const ready = { fallbackReady: true }; + assert.equal(handoffReason("subscription", "rate-limit", null, { ...ready, limitStop: true }), "limit-stop"); + assert.equal(handoffReason("subscription", "rate-limit", 429, { ...ready, limitStop: false }), "", "a 429 with no rejected window"); + assert.equal(handoffReason("subscription", "other", 400, { ...ready, creditStop: true }), "credit-stop"); + assert.equal(handoffReason("subscription", "auth", 401, ready), "auth"); + assert.equal(handoffReason("subscription", "auth", 403, ready), ""); + assert.equal(handoffReason("subscription", "rate-limit", null, { fallbackReady: false, limitStop: true }), "", "no fallback leg"); + assert.equal(handoffReason("fallback", "rate-limit", null, { ...ready, limitStop: true }), "", "the fallback leg is the last"); +}); + +test("only runs that ended with an error are judged, their traces resolved against the result file", () => { + const result = { + cases: [ + { arms: { with: [{ error: null, tracePath: "/k/ok.jsonl" }, { error: "API error", tracePath: "run/2.jsonl" }], without: [{ error: "timed out after 300s", tracePath: "/k/3.jsonl" }] } }, + { arms: { with: [{ error: "no trace kept" }] } }, + ], + }; + assert.deepEqual(erroredTraces(result, "/out"), ["/out/run/2.jsonl", "/k/3.jsonl"]); + assert.deepEqual(erroredTraces(null, "/out"), []); +}); + +test("the first stream that hands off decides, else the first stop class is kept", () => { + const verdicts = { a: ["", ""], b: ["rate-limit", ""], c: ["auth", "auth"], d: ["credit-stop", "credit-stop"] }; + const launcher = { + linesToMessages: (text) => [{ text }], + judgeLeg: ({ executionFile }) => ({ failureClass: verdicts[JSON.parse(readFileSync(executionFile, "utf8"))[0].text][0], resets: null, messages: [{}] }), + isCreditStop: () => false, + }; + const key = { leg: "api-key", fallbackReady: false }; + const streams = (...names) => names.map((text) => ({ text, exitCode: 1 })); + assert.deepEqual(judgeLegStreams(streams("a", "b"), key, launcher), { handoff: false, reason: "", laneClass: "rate-limit" }); + assert.deepEqual(judgeLegStreams(streams("a", "d", "b"), key, launcher), { handoff: true, reason: "credit-stop", laneClass: "credit-stop" }); + assert.deepEqual(judgeLegStreams(streams("a"), key, launcher), { handoff: false, reason: "", laneClass: "" }); + assert.deepEqual(judgeLegStreams([], key, launcher), { handoff: false, reason: "", laneClass: "" }); +}); + +// The classifier the legs check out, fetched at the pin the workflows name, so +// a pin bump that reshapes judgeLeg's return (failureClass, resets, the last +// message's api_error_status) or isCreditStop fails here. The three paths are +// the legs' sparse checkout. +const CLASSIFIER_FILES = [".github/actions/run-claude-cli/lane.cjs", ".github/actions/report-lane-outcome/classify.cjs", ".github/scripts/resolve-npm-tool-version.cjs"]; +const workflowPins = ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"].flatMap((name) => [ + ...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/repository: melodic-software\/ci-workflows\n\s+ref: ([0-9a-f]{40})/g), +].map((m) => m[1])); +const PIN = workflowPins[0]; +const ROOT = tempDir("eval-handoff-ci-workflows-"); +for (const path of CLASSIFIER_FILES) { + const url = `https://raw.githubusercontent.com/melodic-software/ci-workflows/${PIN}/${path}`; + const response = await fetch(url, { signal: AbortSignal.timeout(15000) }); + assert.ok(response.ok, `${url} answered ${response.status}`); + mkdirSync(join(ROOT, path, ".."), { recursive: true }); + writeFileSync(join(ROOT, path), await response.text()); +} +const real = (() => { + const require = createRequire(join(ROOT, ".github/actions/run-claude-cli/lane.cjs")); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + return { judgeLeg, linesToMessages, isCreditStop }; +})(); + +test("every eval leg checks out the classifier at one pin", () => { + assert.equal(workflowPins.length, 6, "three legs in each eval workflow"); + assert.deepEqual([...new Set(workflowPins)], [PIN]); +}); + +test("the pinned judgeLeg returns the fields eval-handoff reads", () => { + const dir = tempDir("eval-handoff-shape-"); + const executionFile = join(dir, "execution.json"); + writeFileSync(executionFile, JSON.stringify(real.linesToMessages(jsonl(LIMITED)))); + const judged = real.judgeLeg({ leg: { leg: "subscription" }, executionFile, exitCode: 1, timedOut: false, maxTurns: "" }); + assert.equal(judged.failureClass, "rate-limit"); + assert.equal(judged.resets, "2100-01-01T00:00:00Z"); + assert.equal(judged.messages.at(-1).api_error_status, 429); + assert.equal(real.isCreditStop(SPEND_LIMIT, { exitCode: 1 }), true); +}); +const jsonl = (messages) => messages.map((m) => JSON.stringify(m)).join("\n"); + +const SPEND_LIMIT_TEXT = "API Error: 400 You have reached your specified workspace API usage limits. You will regain access on 2026-11-01 at 00:00 UTC."; +const SPEND_LIMIT = [ + { type: "assistant", error: "unknown", is_api_error_message: true, api_error_status: 400, message: { model: "", content: [{ type: "text", text: SPEND_LIMIT_TEXT }], usage: { input_tokens: 0, output_tokens: 0 } } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, total_cost_usd: 0, api_error_status: 400, terminal_reason: "api_error", result: SPEND_LIMIT_TEXT, usage: { input_tokens: 0, output_tokens: 0 } }, +]; +const REJECTED_TEXT = "Invalid API key ยท Fix external API key"; +const rejected = (status) => [ + { type: "assistant", error: "authentication_failed", is_api_error_message: true, api_error_status: status, message: { model: "", content: [{ type: "text", text: REJECTED_TEXT }], usage: { input_tokens: 0, output_tokens: 0 } } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, total_cost_usd: 0, api_error_status: status, terminal_reason: "api_error", result: REJECTED_TEXT, usage: { input_tokens: 0, output_tokens: 0 } }, +]; +// A subscription run held by a rejected five-hour window, its reset in the future. +const LIMITED = [ + { type: "system", subtype: "init" }, + { type: "rate_limit_event", rate_limit_info: { status: "rejected", resetsAt: 4102444800, rateLimitType: "five_hour" } }, + { type: "result", subtype: "success", is_error: true, num_turns: 1, api_error_status: 429, result: "limit", usage: { input_tokens: 0, output_tokens: 0 } }, +]; +const CLEAN = [{ type: "system", subtype: "init" }, { type: "result", subtype: "success", is_error: false, num_turns: 3 }]; + +test("the real classifier: a key leg's observed spend-limit stop and 401 hand off, a 403 does not", () => { + const key = { leg: "api-key", fallbackReady: false }; + assert.deepEqual(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], key, real), { handoff: true, reason: "credit-stop", laneClass: "credit-stop" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(rejected(401)), exitCode: 1 }], key, real), { handoff: true, reason: "auth", laneClass: "auth" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(rejected(403)), exitCode: 1 }], key, real), { handoff: false, reason: "", laneClass: "auth" }); + assert.deepEqual(judgeLegStreams([{ text: jsonl(CLEAN), exitCode: 0 }], key, real), { handoff: false, reason: "", laneClass: "" }); +}); + +test("the real classifier: a subscription leg's limit stop and credit stop hand off to the fallback leg", () => { + const sub = { leg: "subscription", fallbackReady: true }; + assert.deepEqual(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], sub, real), { handoff: true, reason: "limit-stop", laneClass: "rate-limit" }); + assert.equal(judgeLegStreams([{ text: jsonl(SPEND_LIMIT), exitCode: 1 }], sub, real).reason, "credit-stop"); + assert.equal(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], { leg: "subscription", fallbackReady: false }, real).handoff, false); + assert.equal(judgeLegStreams([{ text: jsonl(LIMITED), exitCode: 1 }], { leg: "fallback", fallbackReady: true }, real).handoff, false); +}); + +test("the CLI reads the probe and the errored runs, and writes the leg's duration for the lane record", async () => { + const { spawnSync } = await import("node:child_process"); + const dir = tempDir("eval-handoff-cli-"); + mkdirSync(join(dir, "t")); + writeFileSync(join(dir, "probe.jsonl"), jsonl(CLEAN)); + writeFileSync(join(dir, "t/1.jsonl"), jsonl(LIMITED)); + writeFileSync(join(dir, "results.json"), JSON.stringify({ durationSeconds: 7.5, costUsd: 0.25, cases: [{ arms: { with: [{ error: "API error", tracePath: "t/1.jsonl" }] } }] })); + writeFileSync(join(dir, "other.json"), JSON.stringify({ durationSeconds: 2.5, costUsd: 0.5, cases: [{ arms: { with: [{ error: null, tracePath: "t/1.jsonl" }] } }] })); + const results = [join(dir, "other.json"), join(dir, "results.json"), join(dir, "absent.json")].join("\n"); + const run = (leg) => + spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + encoding: "utf8", + env: { LEG: leg, FALLBACK_READY: "true", CI_WORKFLOWS: ROOT, RESULTS: results, PROBE: join(dir, "probe.jsonl"), PROBE_EXIT: "0", LEG_RESULT: join(dir, "leg.json") }, + }); + const sub = run("subscription"); + assert.equal(sub.status, 0, sub.stderr); + assert.equal(sub.stdout, "handoff=true\nhandoff-reason=limit-stop\nlane-class=rate-limit\noutcome=failure\n"); + assert.deepEqual(JSON.parse(readFileSync(join(dir, "leg.json"), "utf8")), [ + { type: "result", subtype: "error_during_execution", is_error: true, duration_ms: 10000, total_cost_usd: 0.75 }, + ]); + assert.equal(run("fallback").stdout, "handoff=false\nhandoff-reason=\nlane-class=rate-limit\noutcome=failure\n"); + assert.equal(run("nope").status, 2); +}); diff --git a/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs b/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs index 923c49ba898..ec3b70ce59d 100644 --- a/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs +++ b/.github/scripts/plugin-eval/hill-climb-workflow.test.mjs @@ -39,8 +39,15 @@ test("only push-branch and publish hold id-token, each for its one lane token; e const writer = file === "maintenance-measure-skill-edit.yml" && Object.hasOwn(WRITERS, name); const permissions = block.match(/\n {4}permissions:\n((?: {6}.+\n)+)/)?.[1] ?? ""; const keys = [...permissions.matchAll(/^ {6}([a-z-]+): (\w+)/gm)].map(([, key, level]) => `${key}: ${level}`); - // The gate also reads this run's re-runner (check-trusted-trigger). - const reads = name === "gate" ? ["contents: read", "actions: read"] : ["contents: read"]; + // The gate also reads this run's re-runner (check-trusted-trigger); a + // key-credit gate reads only the credit-stop markers, and a pass's + // caller grants its pass workflow's gate that read. + const creditGate = name === "propose-key-credit" || (file === "eval-hill-climb-pass.yml" && name === "gate"); + const reads = creditGate + ? ["actions: read"] + : name === "gate" || /^pass-(?:a|a2|b)$/.test(name) + ? ["contents: read", "actions: read"] + : ["contents: read"]; assert.deepEqual(keys, writer ? ["contents: read", "id-token: write"] : reads, `${file} ${name}`); const mints = [...code(block).matchAll(/request-lane-token[^]*?\n {10}activity: ([a-z-]+)\n/g)].map(([, activity]) => activity); assert.deepEqual(mints, writer ? [WRITERS[name]] : [], `${file} ${name} mints`); @@ -125,19 +132,34 @@ test("every ci-workflows action is pinned to one full commit", () => { assert.ok(pins.length >= 2); // report-lane-outcome is pinned where every lane pins it; the rest at one commit. for (const pin of pins) assert.ok(["b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 b8bdd7f 2026-11-04", "6b9c640bf209c0c7e76f1111c5d18c96707348a2 6b9c640 2026-11-02"].includes(pin), pin); + // The classifier each pass leg checks out is run-claude-cli's, at the same commit. + const classifier = [...pass.matchAll(/repository: melodic-software\/ci-workflows\n {10}ref: (\S+) # (.+)/g)].map(([, sha, note]) => `${sha} ${note}`); + assert.deepEqual(classifier, Array(3).fill("b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 b8bdd7f 2026-11-04")); }); -test("the model steps are read-only, on the maintenance token then the fallback token, and frame what they read as data", () => { - assert.equal(main.match(/run-claude-cli@/g).length, 2); +test("the model steps are read-only, on the maintenance key, then its token, then the fallback token, and frame what they read as data", () => { + assert.equal(main.match(/run-claude-cli@/g).length, 3); assert.doesNotMatch(pass, /run-claude-cli@/); const propose = jobs(main).propose; + const key = step(propose, "Propose the edit on the API key"); const leg = step(propose, "Propose the edit"); const fallback = step(propose, "Propose the edit on the fallback token"); + assert.match(key, /api-key: \$\{\{ secrets\.MAINTENANCE_CLAUDE_API_KEY \}\}\n/); + assert.match(key, /if: steps\.prompt\.outcome == 'success' && steps\.key\.outputs\.key == 'true'\n/); assert.match(leg, /subscription-token: \$\{\{ secrets\.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN \}\}\n {10}fallback-ready: \$\{\{ steps\.credential\.outputs\.fallback-ready \}\}\n/); + // The subscription leg runs after the key leg hands off or does not run. + assert.match(leg, /&& \(steps\.run-key\.outcome == 'skipped' \|\| steps\.run-key\.outputs\.handoff == 'true'\)\n/); assert.match(fallback, /fallback-token: \$\{\{ secrets\.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN \}\}\n/); // The fallback leg runs after the subscription leg hands off or finds its token empty, with the fallback token set. - assert.match(fallback, /steps\.credential\.outputs\.fallback-ready == 'true'\n\s+&& \(steps\.run\.outputs\.handoff == 'true' \|\| steps\.credential\.outputs\.present != 'true'\)/); - for (const model of [leg, fallback]) { + assert.match( + fallback, + /steps\.credential\.outputs\.fallback-ready == 'true'\n\s+&& \(steps\.run\.outputs\.handoff == 'true'\n\s+\|\| steps\.credential\.outputs\.present != 'true' && \(steps\.run-key\.outcome == 'skipped' \|\| steps\.run-key\.outputs\.handoff == 'true'\)\)/, + ); + // A credit-stopped key skips the key leg, and only a key credit stop uploads its marker. + assert.match(step(propose, "Check the API key"), /SKIP_KEY: \$\{\{ needs\.propose-key-credit\.outputs\.skip-key \}\}/); + assert.match(step(propose, "Write the key's credit-stop marker"), /'key-credit-stop-maintenance-measure-skill-edit\.yml-propose' && steps\.run-key\.outputs\.handoff-reason == 'credit-stop'/); + assert.match(jobs(main)["propose-key-credit"], /key-job: propose\n/); + for (const model of [key, leg, fallback]) { assert.match(model, /role: maintenance\n/); assert.match(model, /permission-mode: dontAsk\n/); assert.match(model, /tools: \|\n {12}Read\n {12}Grep\n {12}Glob\n {10}allowed-tools:/); diff --git a/.github/scripts/plugin-eval/workflow.test.mjs b/.github/scripts/plugin-eval/workflow.test.mjs index 9c5c704b15d..a4435583a83 100644 --- a/.github/scripts/plugin-eval/workflow.test.mjs +++ b/.github/scripts/plugin-eval/workflow.test.mjs @@ -38,8 +38,14 @@ test("a pull request run proves the policy denies the reads before any case runs for (const call of ["Read /proc/self/environ", 'Read "$RUNNER_ROOT/.runner"', "Grep /proc/self", "Glob /proc/self"]) { assert.ok(probe.includes(`\n denied ${call}\n`), `the probe checks a denied ${call}`); } - assert.match(probe, /if grep -qF "\$CLAUDE_CODE_OAUTH_TOKEN" "\$out"; then\n[^\n]*\n {12}exit 1/); + // The leak check looks for the leg's own credential, whichever leg runs. + assert.match(probe, /credential=\$\{ANTHROPIC_API_KEY:-\$\{CLAUDE_CODE_OAUTH_TOKEN:-\}\}\n {10}\[ -n "\$credential" \] \|\| /); + assert.match(probe, /if grep -qF "\$credential" "\$out"; then\n[^\n]*\n {12}exit 1/); assert.match(probe, /trap 'rm -f "\$out"' EXIT/); + // A probe the credential's own stop ended hands off before the denial checks, never past them. + assert.ok(probe.indexOf("eval-handoff.mjs") < probe.indexOf("denied Read /proc/self/environ")); + assert.ok(probe.indexOf('if grep -qF "$credential"') < probe.indexOf("eval-handoff.mjs"), "the leak check runs before the classifier"); + assert.match(probe, /env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="\$out"/, "the classifier runs without the credential"); assert.doesNotMatch(probe, /cat "\$out"|jq [^\n]*"\$out"[^\n]*\| *tee/); assert.ok(text.indexOf("- name: Install the locked root toolchain") < text.indexOf("- name: Probe that the policy")); assert.ok(text.indexOf("- name: Probe that the policy") < text.indexOf("- name: Run the plugin eval")); @@ -83,13 +89,14 @@ test("a pull request run redacts its summary and uploads no results", () => { assert.match(step("Summarize the results"), /REDACT: \$\{\{ github\.event_name == 'pull_request' && '--redact' \|\| '' \}\}/); assert.match(step("Summarize the results"), /summarize\.mjs "\$results" --plugin "\$name" "\$\{redact\[@\]\}"/); // Only a dispatched run scans and uploads, and the upload needs the scan's clean output. + // A leg that handed off to the next credential uploads nothing. const scan = step("Scan the upload for credentials"); - assert.match(scan, /if: >-\n {10}always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch'\n/); + assert.match(scan, /if: >-\n {10}always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch' && steps\.judge\.outputs\.handoff != 'true'\n/); assert.match(scan, /\n {8}id: scan-upload\n/); assert.match(scan, /\n {10}path: \$\{\{ runner\.temp \}\}\/eval-results\/\n/); assert.match( step("Upload the eval results"), - /if: >-\n {10}\(always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch'\) && steps\.scan-upload\.outputs\.clean == 'true'\n/, + /if: >-\n {10}\(always\(\) && steps\.stage\.outputs\.go == 'true' && github\.event_name == 'workflow_dispatch' && steps\.judge\.outputs\.handoff != 'true'\) && steps\.scan-upload\.outputs\.clean == 'true'\n/, ); assert.ok(text.indexOf("- name: Scan the upload for credentials") < text.indexOf("- name: Upload the eval results")); }); diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml index 658355fbdb0..4d97976555f 100644 --- a/.github/workflows/eval-browser-dispatch.yml +++ b/.github/workflows/eval-browser-dispatch.yml @@ -38,8 +38,17 @@ name: eval-browser-dispatch # whose tree holds no symlink or submodule and whose cases load only the # plugin itself or an unchanged directory inside it. Anything else is refused. # -# The credential is the maintenance role's subscription token. When the secret -# is empty the job ends with a notice and runs nothing. +# The eval runs on the maintenance role's credential chain, one credential per +# job: `eval` on MAINTENANCE_CLAUDE_API_KEY, then `eval-subscription` on +# MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN, then `eval-fallback` on +# FALLBACK_CLAUDE_CODE_OAUTH_TOKEN. A leg hands off on the stops run-claude-cli +# hands off on (eval-handoff.mjs reads the probe's stream and every errored +# run's kept trace with run-claude-cli's own classifier): the key leg on a +# credit stop or a 401 or 402, the subscription leg, when the fallback token is +# set, also on a limit stop; an empty credential hands off too. The leg that +# hands off summarizes and uploads nothing. eval-key-credit skips the key after +# a recent credit stop (ci-workflows check-key-credit). With every credential +# empty the last leg ends with a notice and runs nothing. # # Only an env allowlist (PATH, ANTHROPIC_*, CLAUDE_CODE_*, EVAL_*) reaches an # eval run (code.claude.com/docs/en/plugin-evals, "How runs are isolated", as @@ -119,10 +128,13 @@ concurrency: cancel-in-progress: false jobs: - eval: - # A dispatch on any ref but the default branch skips before any step - # runs, so every case it runs has merged through review. A pull request - # runs only on the `eval: run` label, from this repository, and not as a draft. + # The key's credit gate (ci-workflows check-key-credit, gate.cjs: one + # artifact-list call for the marker the key leg uploads after a credit + # stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour unset) + # starts the eval on the subscription, with no refused key attempt first. + # No secret, no model, no checkout; every failure runs the key (skip-key + # empty). Probing is off. + eval-key-credit: if: >- (github.event_name == 'workflow_dispatch' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) @@ -131,12 +143,64 @@ jobs: && github.event.pull_request.head.repo.full_name == github.repository && !github.event.pull_request.draft) runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: eval + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" + + # The key leg, on the maintenance role's API key alone. A dispatch on any + # ref but the default branch skips before any step runs, so every case it + # runs has merged through review. A pull request runs only on the + # `eval: run` label, from this repository, and not as a draft. A credit + # stop, or a 401 or 402, in the probe or any run hands the eval to + # eval-subscription; an empty key, or one the gate skipped, runs nothing. + eval: + needs: eval-key-credit + if: >- + ${{ !cancelled() && needs.eval-key-credit.result == 'success' && ((github.event_name == 'workflow_dispatch' + && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)) + || (github.event_name == 'pull_request' + && github.event.label.name == 'eval: run' + && github.event.pull_request.head.repo.full_name == github.repository + && !github.event.pull_request.draft)) }} + runs-on: ubuntu-24.04 timeout-minutes: 60 cache-mode: read permissions: contents: read # check out the repository; nothing in this job writes pull-requests: read # check-trusted-trigger reads the pull request actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} env: API_BUDGET_TOKEN: github-token npm_config_audit: "false" @@ -265,24 +329,28 @@ jobs: exit 1 fi - # Only whether the secret is set leaves this step, never its value. + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. - name: Check the credential id: credential env: - PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.eval-key-credit.outputs.skip-key }} run: | - if [ "$PRESENT" = true ]; then - echo "present=true" >>"$GITHUB_OUTPUT" - else - echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; no eval runs." - echo "present=false" >>"$GITHUB_OUTPUT" + run=$PRESENT + if [ "$SKIP_KEY" = true ]; then + run=false + echo "::notice::the credit gate skipped MAINTENANCE_CLAUDE_API_KEY; eval-subscription runs the eval" + elif [ "$PRESENT" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; eval-subscription runs the eval" fi + { echo "present=$PRESENT"; echo "run=$run"; } >>"$GITHUB_OUTPUT" # The plugin under test on a pull request: the label-time head, with full # history so the next step diffs this same commit against its base. A # checkout runs no code from it. - name: Check out the pull request head - if: steps.credential.outputs.present == 'true' && github.event_name == 'pull_request' + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.event.pull_request.head.sha }} @@ -295,7 +363,7 @@ jobs: # dispatch. Each line of `plugins` is a plugins/ path. - name: Choose the plugins id: plugins - if: steps.credential.outputs.present == 'true' + if: steps.credential.outputs.run == 'true' env: EVENT: ${{ github.event_name }} TARGET: ${{ inputs.target }} @@ -470,6 +538,22 @@ jobs: echo "/opt/tools/bin" >> "$GITHUB_PATH" test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + # Proof that the managed policy holds before any pull request case runs, # for each of the three tools a case can use to read a file: one Claude # Code run is told to Read /proc/self/environ and the runner's .runner @@ -480,26 +564,44 @@ jobs: # credential anywhere. Claude Code applies Read deny rules to Glob and # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, # "Read & Edit"), so a Glob or Grep the policy does not stop fails here - # instead of reaching a case. The result can hold what a call returned, + # instead of reaching a case. The stream can hold what a call returned, # so it stays in a file that is never printed and is removed after the - # check. + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. - name: Probe that the policy denies those reads + id: probe if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } out=$(mktemp) trap 'rm -f "$out"' EXIT + status=0 claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ - --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format json >"$out" 2>/dev/null || true - if grep -qF "$CLAUDE_CODE_OAUTH_TOKEN" "$out"; then + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi denied() { - jq -e --arg t "$1" --arg p "$2" 'any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } } denied Read /proc/self/environ @@ -567,9 +669,10 @@ jobs: # run's own sandbox makes read-only to Bash, and within the short length # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). - name: Run the plugin eval - if: steps.stage.outputs.go == 'true' + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} # Strips credentials from Bash subprocesses inside each eval run. # The env-scrub input set false gives "0", the opt-out value Claude # Code's own messages name. @@ -583,6 +686,10 @@ jobs: REAL_SERVERS: ${{ inputs.allow-real-servers }} OUT: ${{ runner.temp }}/eval-results run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi export TMPDIR="$HOME/.ev" mkdir -p "$TMPDIR" args=( @@ -636,6 +743,38 @@ jobs: echo "exit $status" >"$out/exit-code" done <<<"$SUITES" + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT # COMPARABLE per case. A suite with no result fails the job too. # A pull request run prints no case name, error or partial reason: the @@ -644,7 +783,7 @@ jobs: # printed to the job log, with workflow commands off, and kept as # summary.md beside its results.json, which a dispatched run uploads. - name: Summarize the results - if: always() && steps.stage.outputs.go == 'true' + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' env: SUITES: ${{ steps.stage.outputs.suites }} OUT: ${{ runner.temp }}/eval-results @@ -713,7 +852,7 @@ jobs: # omitted when nothing matches. Nothing else from the trace is printed. Trace text is # tool output, so workflow commands are off while it prints. - name: Print Bash error excerpts - if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' env: OUT: ${{ runner.temp }}/eval-results run: | @@ -766,14 +905,14 @@ jobs: - name: Scan the upload for credentials id: scan-upload if: >- - always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] with: path: ${{ runner.temp }}/eval-results/ - name: Upload the eval results if: >- - (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch') && steps.scan-upload.outputs.clean == 'true' + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} @@ -781,6 +920,1581 @@ jobs: if-no-files-found: warn retention-days: 3 + # Read by the next runs' credit gate (eval-key-credit): the artifact its + # `marker` output names (check-key-credit gate.cjs markerName), uploaded + # only after the key leg's credit-stop handoff. The name is written out + # so its retention row claims it. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.eval-key-credit.outputs.marker == 'key-credit-stop-eval-browser-dispatch.yml-eval' && steps.judge.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + id: credit-marker-upload + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-eval-browser-dispatch.yml-eval + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_API_KEY + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: "" + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + + # The subscription leg: the key leg's steps on a fresh runner, on the + # maintenance role's subscription token alone, with no credit-stop marker. + # With FALLBACK_CLAUDE_CODE_OAUTH_TOKEN set, a limit stop, a credit stop, or + # a 401 or 402 hands off to eval-fallback, and so does an empty token. With + # both tokens empty the leg ends with a notice and runs nothing. + # callers.test.mjs holds its steps equal to the key leg's but for the + # credential and the leg record's fallback reason. + eval-subscription: + needs: [eval-key-credit, eval] + if: >- + ${{ !cancelled() && needs.eval-key-credit.result == 'success' + && (needs.eval.outputs.handoff == 'true' || needs.eval.outputs.credential-present == 'false' + || needs.eval-key-credit.outputs.skip-key == 'true') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the repository; nothing in this job writes + pull-requests: read # check-trusted-trigger reads the pull request + actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + API_BUDGET_TOKEN: github-token + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + # npm's fetch timeout and retries, from the one file that sets them. + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Resolve main's tip + id: tip + if: github.event_name == 'pull_request' + env: + TIP_BRANCH: ${{ github.event.repository.default_branch }} + GIT_TERMINAL_PROMPT: "0" + run: | + sha=$(timeout 60 git -C "$RUNNER_TEMP" ls-remote "https://github.com/$GITHUB_REPOSITORY.git" "refs/heads/$TIP_BRANCH" | + awk -v ref="refs/heads/$TIP_BRANCH" '$2 == ref { print $1 }') + [[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "::error::the tip of $TIP_BRANCH did not resolve"; exit 1; } + echo "sha=$sha" >>"$GITHUB_OUTPUT" + # The default branch tip; on a dispatch, the dispatched commit, which the + # job if: holds to the default branch. + - name: Check out the default branch + if: github.event_name != 'pull_request' || steps.tip.outputs.sha != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'pull_request' && steps.tip.outputs.sha || github.sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + # Skips the run while Actions or npm installs report a degraded component, + # with a logged reason; fails open (.github/actions/check-upstream-status). + - name: Check upstream status + id: upstream + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + timeout-minutes: 2 + uses: ./.base/.github/actions/check-upstream-status # zizmor: ignore[self-repository] + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + - name: Stop while upstream is degraded + if: steps.upstream.outputs.degraded == 'true' + env: + REASON: ${{ steps.upstream.outputs.reason }} + run: | + echo "::error::upstream-degraded: ${REASON} on its status page stops this run; dispatch it again once it recovers" + exit 1 + + # check-trusted-trigger needs a PR, and a dispatch names none, so the + # dispatcher is matched against the same list directly, by numeric id: + # it must be a listed human, and the lanes App bot is denied by id, as + # in pr-run-activity-write.yml. A re-run is refused: its re-runner's id + # is only in the run's record, which this job's token cannot read, and + # a login can pass to another account. Dispatch the workflow again + # instead. + - name: Check the dispatcher is a trusted human + if: github.event_name == 'workflow_dispatch' + env: + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + ACTOR_ID: ${{ github.actor_id }} + APP_SENDER_ID: ${{ vars.AUTOMATION_LANES_APP_SENDER_ID }} + run: | + if [ "$GITHUB_RUN_ATTEMPT" != 1 ]; then + echo "::error::rerun-refused: a re-run of this workflow is not supported; dispatch it again" + exit 1 + fi + if ! [[ $APP_SENDER_ID =~ ^[1-9][0-9]*$ ]]; then + echo "::error::the AUTOMATION_LANES_APP_SENDER_ID variable must hold the lanes App bot's numeric user id" + exit 1 + fi + if [ "$ACTOR_ID" = "$APP_SENDER_ID" ]; then + echo "::error::bot-actor: the lanes App bot cannot dispatch this lane" + exit 1 + fi + if ! jq -e --argjson id "$ACTOR_ID" ' + .version == 1 + and any(.actors[]; .id == $id and .kind == "human") + ' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the dispatcher is not a listed human" + exit 1 + fi + + - name: Check the pull request trigger + id: gate + if: github.event_name == 'pull_request' + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + + # check-trusted-trigger accepts any listed actor; the label that spends + # the credential must come from a listed human, so a person opts in to + # each run. + - name: Stop on an untrusted pull request trigger + if: github.event_name == 'pull_request' + env: + PROCEED: ${{ steps.gate.outputs.proceed }} + REASON: ${{ steps.gate.outputs.reason }} + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + SENDER_ID: ${{ github.event.sender.id }} + run: | + if [ "$PROCEED" != true ]; then + echo "::error::the trigger is not trusted (reason: ${REASON:-unreadable})" + exit 1 + fi + if ! jq -e --argjson id "$SENDER_ID" 'any(.actors[]; .id == $id and .kind == "human")' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the 'eval: run' label did not come from a listed human" + exit 1 + fi + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + if [ "$FALLBACK_PRESENT" = true ]; then + echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; eval-fallback runs the eval" + else + echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are empty; no eval runs." + fi + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" + + # The plugin under test on a pull request: the label-time head, with full + # history so the next step diffs this same commit against its base. A + # checkout runs no code from it. + - name: Check out the pull request head + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: .head + fetch-depth: 0 + persist-credentials: false + + # The checked-out head's diff from its merge base with the pull + # request's base chooses the plugins; the target input does on a + # dispatch. Each line of `plugins` is a plugins/ path. + - name: Choose the plugins + id: plugins + if: steps.credential.outputs.run == 'true' + env: + EVENT: ${{ github.event_name }} + TARGET: ${{ inputs.target }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ "$EVENT" = workflow_dispatch ]; then + target=${TARGET%/} + if ! [[ $target =~ ^plugins/[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::target must be a plugin directory, plugins/" + exit 1 + fi + selected=$target + else + merge_base=$(git -C .head merge-base "$BASE_SHA" "$HEAD_SHA") + files=$(git -C .head diff --no-renames --name-only "$merge_base" "$HEAD_SHA") + selected=$(printf '%s\n' "$files" | + node .base/.github/scripts/plugin-eval/select-plugins.mjs --repo .head --base "$merge_base" --head "$HEAD_SHA") || { + echo "::error::the pull request changes more of a selected plugin than its suite may run with the credential" + exit 1 + } + fi + if [ -z "$selected" ]; then + echo "::notice::the pull request changes no plugin's skills, agents or evals; no eval runs." + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "list<>"$GITHUB_OUTPUT" + + # validate-cases.py makes no model call. A FAIL is a case the binary + # rejects at load, so the job stops before any spend. On a deterministic + # run each suite is then copied without its judged cases. + - name: Validate and stage the suites + id: stage + if: steps.plugins.outputs.go == 'true' + env: + LIST: ${{ steps.plugins.outputs.list }} + ROOT: ${{ github.event_name == 'pull_request' && '.head' || '.base' }} + DETERMINISTIC: ${{ github.event_name == 'pull_request' || inputs.deterministic-only }} + run: | + staged="" + while IFS= read -r plugin; do + [ -n "$plugin" ] || continue + dir="$ROOT/$plugin" + eval_dir=$(jq -r '.experimental.evals // "evals"' "$dir/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + if [ ! -d "$dir/$eval_dir" ]; then + echo "::notice::$plugin has no eval suite; skipped." + continue + fi + echo "== validate $plugin/$eval_dir" + if ! python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$dir/$eval_dir"; then + echo "::error::$plugin/$eval_dir does not pass validate-cases.py, so no eval runs" + exit 1 + fi + if [ "$DETERMINISTIC" = true ]; then + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$dir" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::notice::$plugin has no case with deterministic graders only; skipped." + continue + fi + eval_dir=evals-deterministic + fi + staged+="$dir"$'\t'"$eval_dir"$'\n' + done <<<"$LIST" + if [ -z "$staged" ]; then + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "suites<>"$GITHUB_OUTPUT" + + # The tools and Chromium go under /opt, outside HOME, which each eval + # run replaces with a temporary one. Only these directories are made + # while sudo works; no third-party code runs before isolation. + - name: Create the tool directories + if: steps.stage.outputs.go == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools /opt/pw + + # A pull request supplies the prompts, and the eval's Read, Glob and Grep + # run without the Bash sandbox, so nothing else keeps them from the + # credential in /proc//environ or the runner's own files. A + # root-owned managed settings file, which applies inside every eval run + # and which nothing in the run overrides (code.claude.com/docs/en/ + # plugin-evals, "How runs are isolated"; code.claude.com/docs/en/ + # settings, "Settings precedence", as of 2026-10-27), denies those + # paths. The runner's install directory is read from the Runner.Worker + # process above this step, not guessed; the job's _temp and _actions + # directories are denied beside it. Written while sudo is still there; + # the job stops if the runner directory is not found or the file cannot + # be written or read back. The probe step below proves the denial. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions." + + # Installs the bubblewrap and socat the Bash sandbox needs, lifts the + # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo + # and docker access for the rest of the job. + - name: Isolate the runner + if: steps.stage.outputs.go == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + # Every binary this job runs comes from the locked root toolchain. Its + # install scripts run here, after sudo and docker access are gone. + - name: Install the locked root toolchain + timeout-minutes: 5 + if: steps.stage.outputs.go == 'true' + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # The guardrails hooks read every Bash command through shfmt and refuse the + # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin + # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run + # replaces HOME. + - name: Install pinned shfmt for guardrails + if: steps.stage.outputs.go == 'true' && contains(steps.plugins.outputs.list, 'plugins/guardrails') + env: + SHFMT_VERSION: v3.14.1 + SHFMT_SHA256: 76e77641faa025814b77f153b29796b8e6fa2fca03e0c76a691608b86c7ea7bf + run: | + mkdir -p /opt/tools/bin + curl -fSL --no-progress-meter --retry 3 --retry-all-errors -o /opt/tools/bin/shfmt \ + "https://github.com/mvdan/sh/releases/download/$SHFMT_VERSION/shfmt_${SHFMT_VERSION}_linux_amd64" + echo "$SHFMT_SHA256 /opt/tools/bin/shfmt" | sha256sum -c - + chmod +x /opt/tools/bin/shfmt + echo "/opt/tools/bin" >> "$GITHUB_PATH" + test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # Proof that the managed policy holds before any pull request case runs, + # for each of the three tools a case can use to read a file: one Claude + # Code run is told to Read /proc/self/environ and the runner's .runner + # file, Grep /proc/self/environ, and Glob for environ under /proc/self. The job stops + # unless the result's permission_denials (code.claude.com/docs/en/ + # agent-sdk/typescript, "SDKResultMessage", as of 2026-10-28) records + # each of the four calls as denied, or if the result holds the + # credential anywhere. Claude Code applies Read deny rules to Glob and + # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, + # "Read & Edit"), so a Glob or Grep the policy does not stop fails here + # instead of reaching a case. The stream can hold what a call returned, + # so it stays in a file that is never printed and is removed after the + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. + - name: Probe that the policy denies those reads + id: probe + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no pull request case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + echo "The managed policy denied the probe's Read, Grep and Glob calls." + + # Chromium comes from the playwright that the locked playwright-cli + # and Playwright MCP server share. Without sudo there is no --with-deps: the hosted image already + # carries a browser's system libraries. A pull request run grants no + # Bash, so it needs no browser: it passes no --allow-tools, a case + # cannot widen that grant (plugins/evals/skills/plugin-eval/SKILL.md, + # the read-only suites paragraph), and select-plugins.mjs refuses a + # changed case's allowed_tools. + - name: Install Chromium + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + env: + PLAYWRIGHT_BROWSERS_PATH: /opt/pw + run: | + node /opt/tools/node_modules/playwright/cli.js install chromium + command -v playwright-cli claude + + # Every Bash call in an eval run goes through Claude Code's bubblewrap + # sandbox (code.claude.com/docs/en/sandboxing, "Set up Linux and WSL2", + # as of 2026-10-10). Starting bwrap here as the runner user, with the + # whole filesystem (so /opt/tools and the setup-node toolcache) bound + # read-only, fails the job with bwrap's own error if bwrap cannot start + # a sandbox, before any eval spends tokens. Its flags are not Claude + # Code's own, so a pass does not prove the full sandbox starts. + - name: Check bwrap starts a sandbox + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + run: | + fail() { + local msg=${2//'%'/'%25'} + msg=${msg//$'\r'/'%0D'} + echo "::error::$1: ${msg//$'\n'/'%0A'}" + exit 1 + } + for tool in bwrap socat; do + command -v "$tool" || fail "$tool is not on PATH" "the Bash sandbox needs it" + done + sysctl kernel.apparmor_restrict_unprivileged_userns || true + sandbox=(bwrap --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --unshare-all --die-with-parent) + if ! out=$("${sandbox[@]}" true 2>&1); then + fail "bwrap cannot start a sandbox as the runner user" "$out" + fi + if ! out=$("${sandbox[@]}" playwright-cli --version 2>&1); then + fail "playwright-cli does not start inside bwrap" "$out" + fi + echo "playwright-cli inside bwrap: $out" + + - name: Record that env scrub is off + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && !inputs.env-scrub + env: + DISPATCHER: ${{ github.actor }} + run: echo "::warning::CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is off for this run; Bash subprocesses can see the Claude credentials. Dispatched by ${DISPATCHER}." + + # One `claude plugin eval` per staged suite, each under its own + # --max-cost-usd. Its exit code is overloaded, so the summary step reads + # each JSON result instead. --keep-temp keeps each run's trace on the + # runner, outside the uploaded results, for the summary's health lines. + # A pull request run grants no tool, runs no scaffold, keeps env scrub on + # and uses each case's own run count. TMPDIR sits outside /tmp, which the + # run's own sandbox makes read-only to Bash, and within the short length + # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). + - name: Run the plugin eval + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + # Strips credentials from Bash subprocesses inside each eval run. + # The env-scrub input set false gives "0", the opt-out value Claude + # Code's own messages name. + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: ${{ (github.event_name == 'pull_request' || inputs.env-scrub) && '1' || '0' }} + SUITES: ${{ steps.stage.outputs.suites }} + DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} + CASE: ${{ inputs.case }} + TAGS: ${{ inputs.tag }} + RUNS: ${{ inputs.runs }} + ALLOW_TOOLS: ${{ inputs.allow-tools }} + REAL_SERVERS: ${{ inputs.allow-real-servers }} + OUT: ${{ runner.temp }}/eval-results + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" + args=( + --trust-plugin + --no-publish + --model claude-sonnet-5-5 + --judge-model claude-opus-5-5 + --max-cost-usd 5 + --keep-temp + ) + if [ "$DISPATCH" = true ]; then + if ! [[ $RUNS =~ ^[1-9][0-9]*$ ]]; then + echo "::error::runs must be a positive integer" + exit 1 + fi + IFS=, read -ra tools <<< "$ALLOW_TOOLS" + read -ra tags <<< "$TAGS" + for value in "${tools[@]}" "${tags[@]}" "$CASE"; do + case "$value" in + -*) echo "::error::a case, tag or tool must not start with -"; exit 1 ;; + esac + done + args+=(--scaffold --runs "$RUNS") + if [ "$REAL_SERVERS" = true ]; then + args+=(--allow-real-servers) + fi + if [ "${#tools[@]}" -gt 0 ]; then + args+=(--allow-tools "${tools[@]}") + fi + if [ "${#tags[@]}" -gt 0 ]; then + args+=(--tag "${tags[@]}") + fi + if [ -n "$CASE" ]; then + args+=(--case "$CASE") + fi + fi + while IFS=$'\t' read -r dir eval_dir; do + [ -n "$dir" ] || continue + out="$OUT/$(basename "$dir")" + mkdir -p "$out" + echo "== claude plugin eval $dir --eval-dir $eval_dir" + status=0 + if [ "$DISPATCH" = true ]; then + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" || status=$? + else + # A pull request run's console output can carry model text from + # the PR's cases, and the job log is public: it goes to a file + # that nothing uploads. + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" >"$out/console.log" 2>&1 || status=$? + fi + echo "exit $status" >"$out/exit-code" + done <<<"$SUITES" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT + # COMPARABLE per case. A suite with no result fails the job too. + # A pull request run prints no case name, error or partial reason: the + # model output those can carry would land in the public job summary. + # The REST API returns no job summary, so each suite's summary is also + # printed to the job log, with workflow commands off, and kept as + # summary.md beside its results.json, which a dispatched run uploads. + - name: Summarize the results + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + REDACT: ${{ github.event_name == 'pull_request' && '--redact' || '' }} + DISPATCHED: ${{ github.event_name == 'workflow_dispatch' }} + RUNS: ${{ inputs.runs }} + run: | + failed=0 + redact=() + if [ -n "$REDACT" ]; then redact=("$REDACT"); fi + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + name=$(basename "$dir") + results="$OUT/$name/results.json" + if [ ! -f "$results" ]; then + why=$(cat "$OUT/$name/exit-code" 2>/dev/null || echo "the run did not start") + printf '### Plugin eval: %s\n\nNo result was written (%s).\n\nResult: NOT COMPARABLE\n\n' \ + "$name" "$why" >>"$GITHUB_STEP_SUMMARY" + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::No result was written (${why//[$'\r\n%']/ })." + failed=1 + continue + fi + summary="$OUT/$name/summary.md" + annotation="$OUT/$name/annotation.txt" + node .base/.github/scripts/plugin-eval/summarize.mjs "$results" --plugin "$name" "${redact[@]}" --annotation "$annotation" >"$summary" || failed=1 + # A dispatched run uploads its results, but not the traces --keep-temp + # keeps here, so the validity verdict is read here and only its + # count-only bash ran line (the line just before the verdict, in its exact format) and its last line are kept (its detail + # lines can quote a trace), as eval-hill-climb-pass.yml does. + # noise-report reads only results.json, which the upload already + # holds. A dispatched run always scaffolds. + if [ "$DISPATCHED" = true ]; then + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$results" --runs "$RUNS" --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + verdict=$(printf '%s\n' "$validity" | tail -n 1) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$verdict" >"$OUT/$name/validity.txt" + python3 .base/plugins/evals/skills/plugin-eval/scripts/noise-report.py "$results" >"$OUT/$name/noise.txt" 2>&1 || true + printf '\nValidity: %s\n' "$verdict" >>"$summary" + if [ -n "$bash_ran" ]; then printf '\n%s\n' "$bash_ran" >>"$summary"; fi + fi + cat "$summary" >>"$GITHUB_STEP_SUMMARY" + echo >>"$GITHUB_STEP_SUMMARY" + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + cat "$summary" + echo "::$token::" + # summarize.mjs escapes the command it writes; it prints outside + # the stop-commands block so the runner reads it. + # No file at all: summarize.mjs could not read the result (exit 2). + if [ -s "$annotation" ]; then + head -n 1 "$annotation" + elif [ ! -e "$annotation" ]; then + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::The result could not be read." + fi + done <<<"$SUITES" + exit "$failed" + + # Opt-in because the job log of this public repository is public. Prints, + # per run, at most 20 Bash calls whose tool_result has is_error true: the + # command cut to 300 characters, and the result text cut to its first 120 + # and last 2500 characters when longer than 2620 (the end holds the daemon + # log), each JSON-quoted onto one line. A second line, `key:`, holds the lines of the + # full result that match FATAL, Check failed, ERROR:, sandbox, permission or errno + # patterns (the cause line that a long crash stack pushes out of the tail), minus the + # known .bashrc noise, de-duplicated, at most 15, each cut to 300 characters; it is + # omitted when nothing matches. Nothing else from the trace is printed. Trace text is + # tool output, so workflow commands are off while it prints. + - name: Print Bash error excerpts + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + OUT: ${{ runner.temp }}/eval-results + run: | + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + for RESULTS in "$OUT"/*/results.json; do + if [ ! -f "$RESULTS" ]; then + echo "no results.json, so no traces to read" + continue + fi + jq -r '.cases[]? as $c | $c.arms // {} | to_entries[] | .key as $arm + | .value // [] | to_entries[] + | [($c.name // $c.dir // "case"), $arm, (.key + 1 | tostring), (.value.tracePath // "")] + | @tsv' "$RESULTS" | + while IFS=$'\t' read -r name arm run trace; do + echo "== $name, $arm arm, run $run" + if [ ! -f "$trace" ]; then + echo " trace not found" + continue + fi + jq -nrR ' + def tail_cut: if length > 2620 then .[:120] + " โ€ฆ " + .[-2500:] else . end; + def key_lines: + [split("\n")[] + | select(test("FATAL|Check failed|ERROR:|No usable sandbox|Permission denied|Operation not permitted|EACCES|EPERM|ENOENT|errno") + and (contains(".bashrc: Permission denied") | not)) + | .[:300]] + | reduce .[] as $l ([]; if index([$l]) then . else . + [$l] end) + | .[:15] | join("\n"); + [inputs | fromjson? | objects] as $m + | ([$m[] | select(.type == "assistant") | .message.content[]? | objects + | select(.type == "tool_use" and .name == "Bash" and (.id | type) == "string") + | {key: .id, value: (.input.command // "" | tostring)}] | from_entries) as $bash + | [$m[] | select(.type == "user") | .message.content[]? | objects + | select(.type == "tool_result" and .is_error == true) + | (.tool_use_id // "" | tostring) as $id + | select($bash | has($id)) + | {cmd: $bash[$id], + out: (.content | if type == "array" then map(objects | select(.type == "text") | .text | tostring) | join("\n") else tostring end)}] + | " \(length) Bash error(s), showing up to 20", + (.[:20][] | " $ \(.cmd[:300] | tojson)\n -> \(.out | tail_cut | tojson)\(.out | key_lines | if . == "" then "" else "\n key: \(tojson)" end)") + ' "$trace" || echo " could not read the trace" + done || echo "could not read $RESULTS" + done + echo "::$token::" + + # Only a dispatched run, whose cases merged through review, uploads its + # results: a pull request's prompts could steer model output into a + # public artifact. + - name: Scan the upload for credentials + id: scan-upload + if: >- + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/eval-results/ + + - name: Upload the eval results + if: >- + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/eval-results/ + if-no-files-found: warn + retention-days: 3 + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.eval.outputs.handoff-reason || needs.eval-key-credit.outputs.skip-key == 'true' && 'credit-stop' || needs.eval.outputs.credential-present == 'false' && 'no-key' || '' }} + + - name: Log the API budget at the end + if: always() + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/log-api-budget # zizmor: ignore[self-repository] + with: + tokens: github-token + + # The fallback leg: the same steps on the central fallback token, after + # the subscription leg handed off or found its token empty, and only when + # the fallback token is set. It is the last leg and never hands off. + eval-fallback: + needs: [eval-key-credit, eval-subscription] + if: >- + ${{ !cancelled() && needs.eval-subscription.outputs.fallback-ready == 'true' + && (needs.eval-subscription.outputs.handoff == 'true' || needs.eval-subscription.outputs.credential-present == 'false') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the repository; nothing in this job writes + pull-requests: read # check-trusted-trigger reads the pull request + actions: read # check-trusted-trigger reads this run's re-runner + outputs: + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + API_BUDGET_TOKEN: github-token + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + # npm's fetch timeout and retries, from the one file that sets them. + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Resolve main's tip + id: tip + if: github.event_name == 'pull_request' + env: + TIP_BRANCH: ${{ github.event.repository.default_branch }} + GIT_TERMINAL_PROMPT: "0" + run: | + sha=$(timeout 60 git -C "$RUNNER_TEMP" ls-remote "https://github.com/$GITHUB_REPOSITORY.git" "refs/heads/$TIP_BRANCH" | + awk -v ref="refs/heads/$TIP_BRANCH" '$2 == ref { print $1 }') + [[ $sha =~ ^[0-9a-f]{40}$ ]] || { echo "::error::the tip of $TIP_BRANCH did not resolve"; exit 1; } + echo "sha=$sha" >>"$GITHUB_OUTPUT" + # The default branch tip; on a dispatch, the dispatched commit, which the + # job if: holds to the default branch. + - name: Check out the default branch + if: github.event_name != 'pull_request' || steps.tip.outputs.sha != '' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event_name == 'pull_request' && steps.tip.outputs.sha || github.sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + # Skips the run while Actions or npm installs report a degraded component, + # with a logged reason; fails open (.github/actions/check-upstream-status). + - name: Check upstream status + id: upstream + if: steps.kill.outputs.proceed == 'true' + continue-on-error: true + timeout-minutes: 2 + uses: ./.base/.github/actions/check-upstream-status # zizmor: ignore[self-repository] + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + - name: Stop while upstream is degraded + if: steps.upstream.outputs.degraded == 'true' + env: + REASON: ${{ steps.upstream.outputs.reason }} + run: | + echo "::error::upstream-degraded: ${REASON} on its status page stops this run; dispatch it again once it recovers" + exit 1 + + # check-trusted-trigger needs a PR, and a dispatch names none, so the + # dispatcher is matched against the same list directly, by numeric id: + # it must be a listed human, and the lanes App bot is denied by id, as + # in pr-run-activity-write.yml. A re-run is refused: its re-runner's id + # is only in the run's record, which this job's token cannot read, and + # a login can pass to another account. Dispatch the workflow again + # instead. + - name: Check the dispatcher is a trusted human + if: github.event_name == 'workflow_dispatch' + env: + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + ACTOR_ID: ${{ github.actor_id }} + APP_SENDER_ID: ${{ vars.AUTOMATION_LANES_APP_SENDER_ID }} + run: | + if [ "$GITHUB_RUN_ATTEMPT" != 1 ]; then + echo "::error::rerun-refused: a re-run of this workflow is not supported; dispatch it again" + exit 1 + fi + if ! [[ $APP_SENDER_ID =~ ^[1-9][0-9]*$ ]]; then + echo "::error::the AUTOMATION_LANES_APP_SENDER_ID variable must hold the lanes App bot's numeric user id" + exit 1 + fi + if [ "$ACTOR_ID" = "$APP_SENDER_ID" ]; then + echo "::error::bot-actor: the lanes App bot cannot dispatch this lane" + exit 1 + fi + if ! jq -e --argjson id "$ACTOR_ID" ' + .version == 1 + and any(.actors[]; .id == $id and .kind == "human") + ' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the dispatcher is not a listed human" + exit 1 + fi + + - name: Check the pull request trigger + id: gate + if: github.event_name == 'pull_request' + uses: ./.base/.github/actions/check-trusted-trigger # zizmor: ignore[self-repository] + with: + trusted-actors-path: .base/.github/standards/trusted-actors/trusted-actors.json + triggering-actor: ${{ github.triggering_actor }} + + # check-trusted-trigger accepts any listed actor; the label that spends + # the credential must come from a listed human, so a person opts in to + # each run. + - name: Stop on an untrusted pull request trigger + if: github.event_name == 'pull_request' + env: + PROCEED: ${{ steps.gate.outputs.proceed }} + REASON: ${{ steps.gate.outputs.reason }} + LIST: .base/.github/standards/trusted-actors/trusted-actors.json + SENDER_ID: ${{ github.event.sender.id }} + run: | + if [ "$PROCEED" != true ]; then + echo "::error::the trigger is not trusted (reason: ${REASON:-unreadable})" + exit 1 + fi + if ! jq -e --argjson id "$SENDER_ID" 'any(.actors[]; .id == $id and .kind == "human")' "$LIST" >/dev/null; then + echo "::error::untrusted-actor: the 'eval: run' label did not come from a listed human" + exit 1 + fi + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the eval. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + echo "::notice::FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is empty; no eval runs." + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; } >>"$GITHUB_OUTPUT" + + # The plugin under test on a pull request: the label-time head, with full + # history so the next step diffs this same commit against its base. A + # checkout runs no code from it. + - name: Check out the pull request head + if: steps.credential.outputs.run == 'true' && github.event_name == 'pull_request' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: .head + fetch-depth: 0 + persist-credentials: false + + # The checked-out head's diff from its merge base with the pull + # request's base chooses the plugins; the target input does on a + # dispatch. Each line of `plugins` is a plugins/ path. + - name: Choose the plugins + id: plugins + if: steps.credential.outputs.run == 'true' + env: + EVENT: ${{ github.event_name }} + TARGET: ${{ inputs.target }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + if [ "$EVENT" = workflow_dispatch ]; then + target=${TARGET%/} + if ! [[ $target =~ ^plugins/[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::target must be a plugin directory, plugins/" + exit 1 + fi + selected=$target + else + merge_base=$(git -C .head merge-base "$BASE_SHA" "$HEAD_SHA") + files=$(git -C .head diff --no-renames --name-only "$merge_base" "$HEAD_SHA") + selected=$(printf '%s\n' "$files" | + node .base/.github/scripts/plugin-eval/select-plugins.mjs --repo .head --base "$merge_base" --head "$HEAD_SHA") || { + echo "::error::the pull request changes more of a selected plugin than its suite may run with the credential" + exit 1 + } + fi + if [ -z "$selected" ]; then + echo "::notice::the pull request changes no plugin's skills, agents or evals; no eval runs." + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "list<>"$GITHUB_OUTPUT" + + # validate-cases.py makes no model call. A FAIL is a case the binary + # rejects at load, so the job stops before any spend. On a deterministic + # run each suite is then copied without its judged cases. + - name: Validate and stage the suites + id: stage + if: steps.plugins.outputs.go == 'true' + env: + LIST: ${{ steps.plugins.outputs.list }} + ROOT: ${{ github.event_name == 'pull_request' && '.head' || '.base' }} + DETERMINISTIC: ${{ github.event_name == 'pull_request' || inputs.deterministic-only }} + run: | + staged="" + while IFS= read -r plugin; do + [ -n "$plugin" ] || continue + dir="$ROOT/$plugin" + eval_dir=$(jq -r '.experimental.evals // "evals"' "$dir/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + if [ ! -d "$dir/$eval_dir" ]; then + echo "::notice::$plugin has no eval suite; skipped." + continue + fi + echo "== validate $plugin/$eval_dir" + if ! python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$dir/$eval_dir"; then + echo "::error::$plugin/$eval_dir does not pass validate-cases.py, so no eval runs" + exit 1 + fi + if [ "$DETERMINISTIC" = true ]; then + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$dir" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::notice::$plugin has no case with deterministic graders only; skipped." + continue + fi + eval_dir=evals-deterministic + fi + staged+="$dir"$'\t'"$eval_dir"$'\n' + done <<<"$LIST" + if [ -z "$staged" ]; then + echo "go=false" >>"$GITHUB_OUTPUT" + exit 0 + fi + { + echo "go=true" + echo "suites<>"$GITHUB_OUTPUT" + + # The tools and Chromium go under /opt, outside HOME, which each eval + # run replaces with a temporary one. Only these directories are made + # while sudo works; no third-party code runs before isolation. + - name: Create the tool directories + if: steps.stage.outputs.go == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools /opt/pw + + # A pull request supplies the prompts, and the eval's Read, Glob and Grep + # run without the Bash sandbox, so nothing else keeps them from the + # credential in /proc//environ or the runner's own files. A + # root-owned managed settings file, which applies inside every eval run + # and which nothing in the run overrides (code.claude.com/docs/en/ + # plugin-evals, "How runs are isolated"; code.claude.com/docs/en/ + # settings, "Settings precedence", as of 2026-10-27), denies those + # paths. The runner's install directory is read from the Runner.Worker + # process above this step, not guessed; the job's _temp and _actions + # directories are denied beside it. Written while sudo is still there; + # the job stops if the runner directory is not found or the file cannot + # be written or read back. The probe step below proves the denial. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + echo "Denied the runner directory $runner_root and $work/_temp, $work/_actions." + + # Installs the bubblewrap and socat the Bash sandbox needs, lifts the + # Ubuntu 24.04 AppArmor user-namespace restriction, then removes sudo + # and docker access for the rest of the job. + - name: Isolate the runner + if: steps.stage.outputs.go == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + # Every binary this job runs comes from the locked root toolchain. Its + # install scripts run here, after sudo and docker access are gone. + - name: Install the locked root toolchain + timeout-minutes: 5 + if: steps.stage.outputs.go == 'true' + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # The guardrails hooks read every Bash command through shfmt and refuse the + # call without it, so a guardrails eval needs it on PATH. Same v3.14.1 pin + # and SHA-256 as pr-require-checks.yml. In /opt/tools, since each eval run + # replaces HOME. + - name: Install pinned shfmt for guardrails + if: steps.stage.outputs.go == 'true' && contains(steps.plugins.outputs.list, 'plugins/guardrails') + env: + SHFMT_VERSION: v3.14.1 + SHFMT_SHA256: 76e77641faa025814b77f153b29796b8e6fa2fca03e0c76a691608b86c7ea7bf + run: | + mkdir -p /opt/tools/bin + curl -fSL --no-progress-meter --retry 3 --retry-all-errors -o /opt/tools/bin/shfmt \ + "https://github.com/mvdan/sh/releases/download/$SHFMT_VERSION/shfmt_${SHFMT_VERSION}_linux_amd64" + echo "$SHFMT_SHA256 /opt/tools/bin/shfmt" | sha256sum -c - + chmod +x /opt/tools/bin/shfmt + echo "/opt/tools/bin" >> "$GITHUB_PATH" + test "$(/opt/tools/bin/shfmt --version)" = "$SHFMT_VERSION" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.stage.outputs.go == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # Proof that the managed policy holds before any pull request case runs, + # for each of the three tools a case can use to read a file: one Claude + # Code run is told to Read /proc/self/environ and the runner's .runner + # file, Grep /proc/self/environ, and Glob for environ under /proc/self. The job stops + # unless the result's permission_denials (code.claude.com/docs/en/ + # agent-sdk/typescript, "SDKResultMessage", as of 2026-10-28) records + # each of the four calls as denied, or if the result holds the + # credential anywhere. Claude Code applies Read deny rules to Glob and + # Grep only on a best-effort basis (code.claude.com/docs/en/permissions, + # "Read & Edit"), so a Glob or Grep the policy does not stop fails here + # instead of reaching a case. The stream can hold what a call returned, + # so it stays in a file that is never printed and is removed after the + # check. A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the eval. + - name: Probe that the policy denies those reads + id: probe + if: steps.stage.outputs.go == 'true' && github.event_name == 'pull_request' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no pull request case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no pull request case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + echo "The managed policy denied the probe's Read, Grep and Glob calls." + + # Chromium comes from the playwright that the locked playwright-cli + # and Playwright MCP server share. Without sudo there is no --with-deps: the hosted image already + # carries a browser's system libraries. A pull request run grants no + # Bash, so it needs no browser: it passes no --allow-tools, a case + # cannot widen that grant (plugins/evals/skills/plugin-eval/SKILL.md, + # the read-only suites paragraph), and select-plugins.mjs refuses a + # changed case's allowed_tools. + - name: Install Chromium + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + env: + PLAYWRIGHT_BROWSERS_PATH: /opt/pw + run: | + node /opt/tools/node_modules/playwright/cli.js install chromium + command -v playwright-cli claude + + # Every Bash call in an eval run goes through Claude Code's bubblewrap + # sandbox (code.claude.com/docs/en/sandboxing, "Set up Linux and WSL2", + # as of 2026-10-10). Starting bwrap here as the runner user, with the + # whole filesystem (so /opt/tools and the setup-node toolcache) bound + # read-only, fails the job with bwrap's own error if bwrap cannot start + # a sandbox, before any eval spends tokens. Its flags are not Claude + # Code's own, so a pass does not prove the full sandbox starts. + - name: Check bwrap starts a sandbox + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' + run: | + fail() { + local msg=${2//'%'/'%25'} + msg=${msg//$'\r'/'%0D'} + echo "::error::$1: ${msg//$'\n'/'%0A'}" + exit 1 + } + for tool in bwrap socat; do + command -v "$tool" || fail "$tool is not on PATH" "the Bash sandbox needs it" + done + sysctl kernel.apparmor_restrict_unprivileged_userns || true + sandbox=(bwrap --ro-bind / / --dev /dev --proc /proc --tmpfs /tmp --unshare-all --die-with-parent) + if ! out=$("${sandbox[@]}" true 2>&1); then + fail "bwrap cannot start a sandbox as the runner user" "$out" + fi + if ! out=$("${sandbox[@]}" playwright-cli --version 2>&1); then + fail "playwright-cli does not start inside bwrap" "$out" + fi + echo "playwright-cli inside bwrap: $out" + + - name: Record that env scrub is off + if: steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && !inputs.env-scrub + env: + DISPATCHER: ${{ github.actor }} + run: echo "::warning::CLAUDE_CODE_SUBPROCESS_ENV_SCRUB is off for this run; Bash subprocesses can see the Claude credentials. Dispatched by ${DISPATCHER}." + + # One `claude plugin eval` per staged suite, each under its own + # --max-cost-usd. Its exit code is overloaded, so the summary step reads + # each JSON result instead. --keep-temp keeps each run's trace on the + # runner, outside the uploaded results, for the summary's health lines. + # A pull request run grants no tool, runs no scaffold, keeps env scrub on + # and uses each case's own run count. TMPDIR sits outside /tmp, which the + # run's own sandbox makes read-only to Bash, and within the short length + # the sandbox's sockets need (plugins/evals/skills/plugin-eval/reference/sandboxes.md). + - name: Run the plugin eval + id: eval + if: steps.stage.outputs.go == 'true' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + # Strips credentials from Bash subprocesses inside each eval run. + # The env-scrub input set false gives "0", the opt-out value Claude + # Code's own messages name. + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: ${{ (github.event_name == 'pull_request' || inputs.env-scrub) && '1' || '0' }} + SUITES: ${{ steps.stage.outputs.suites }} + DISPATCH: ${{ github.event_name == 'workflow_dispatch' }} + CASE: ${{ inputs.case }} + TAGS: ${{ inputs.tag }} + RUNS: ${{ inputs.runs }} + ALLOW_TOOLS: ${{ inputs.allow-tools }} + REAL_SERVERS: ${{ inputs.allow-real-servers }} + OUT: ${{ runner.temp }}/eval-results + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" + args=( + --trust-plugin + --no-publish + --model claude-sonnet-5-5 + --judge-model claude-opus-5-5 + --max-cost-usd 5 + --keep-temp + ) + if [ "$DISPATCH" = true ]; then + if ! [[ $RUNS =~ ^[1-9][0-9]*$ ]]; then + echo "::error::runs must be a positive integer" + exit 1 + fi + IFS=, read -ra tools <<< "$ALLOW_TOOLS" + read -ra tags <<< "$TAGS" + for value in "${tools[@]}" "${tags[@]}" "$CASE"; do + case "$value" in + -*) echo "::error::a case, tag or tool must not start with -"; exit 1 ;; + esac + done + args+=(--scaffold --runs "$RUNS") + if [ "$REAL_SERVERS" = true ]; then + args+=(--allow-real-servers) + fi + if [ "${#tools[@]}" -gt 0 ]; then + args+=(--allow-tools "${tools[@]}") + fi + if [ "${#tags[@]}" -gt 0 ]; then + args+=(--tag "${tags[@]}") + fi + if [ -n "$CASE" ]; then + args+=(--case "$CASE") + fi + fi + while IFS=$'\t' read -r dir eval_dir; do + [ -n "$dir" ] || continue + out="$OUT/$(basename "$dir")" + mkdir -p "$out" + echo "== claude plugin eval $dir --eval-dir $eval_dir" + status=0 + if [ "$DISPATCH" = true ]; then + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" || status=$? + else + # A pull request run's console output can carry model text from + # the PR's cases, and the job log is public: it goes to a file + # that nothing uploads. + claude plugin eval "$dir" --eval-dir "$eval_dir" --output-dir "$out" --json "$out/results.json" "${args[@]}" >"$out/console.log" 2>&1 || status=$? + fi + echo "exit $status" >"$out/exit-code" + done <<<"$SUITES" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suites' durationSeconds and costUsd, for the lane record. A leg + # that hands off summarizes and uploads nothing: the next leg reruns the + # eval. + - name: Judge the leg + id: judge + if: always() && steps.stage.outputs.go == 'true' && (steps.probe.outcome == 'success' || steps.probe.outcome == 'skipped') + env: + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + results="" + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + results+="$OUT/$(basename "$dir")/results.json"$'\n' + done <<<"$SUITES" + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The verdict of record: summarize.mjs per result, PASS, FAIL or NOT + # COMPARABLE per case. A suite with no result fails the job too. + # A pull request run prints no case name, error or partial reason: the + # model output those can carry would land in the public job summary. + # The REST API returns no job summary, so each suite's summary is also + # printed to the job log, with workflow commands off, and kept as + # summary.md beside its results.json, which a dispatched run uploads. + - name: Summarize the results + if: always() && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + SUITES: ${{ steps.stage.outputs.suites }} + OUT: ${{ runner.temp }}/eval-results + REDACT: ${{ github.event_name == 'pull_request' && '--redact' || '' }} + DISPATCHED: ${{ github.event_name == 'workflow_dispatch' }} + RUNS: ${{ inputs.runs }} + run: | + failed=0 + redact=() + if [ -n "$REDACT" ]; then redact=("$REDACT"); fi + while IFS=$'\t' read -r dir _; do + [ -n "$dir" ] || continue + name=$(basename "$dir") + results="$OUT/$name/results.json" + if [ ! -f "$results" ]; then + why=$(cat "$OUT/$name/exit-code" 2>/dev/null || echo "the run did not start") + printf '### Plugin eval: %s\n\nNo result was written (%s).\n\nResult: NOT COMPARABLE\n\n' \ + "$name" "$why" >>"$GITHUB_STEP_SUMMARY" + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::No result was written (${why//[$'\r\n%']/ })." + failed=1 + continue + fi + summary="$OUT/$name/summary.md" + annotation="$OUT/$name/annotation.txt" + node .base/.github/scripts/plugin-eval/summarize.mjs "$results" --plugin "$name" "${redact[@]}" --annotation "$annotation" >"$summary" || failed=1 + # A dispatched run uploads its results, but not the traces --keep-temp + # keeps here, so the validity verdict is read here and only its + # count-only bash ran line (the line just before the verdict, in its exact format) and its last line are kept (its detail + # lines can quote a trace), as eval-hill-climb-pass.yml does. + # noise-report reads only results.json, which the upload already + # holds. A dispatched run always scaffolds. + if [ "$DISPATCHED" = true ]; then + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$results" --runs "$RUNS" --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + verdict=$(printf '%s\n' "$validity" | tail -n 1) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$verdict" >"$OUT/$name/validity.txt" + python3 .base/plugins/evals/skills/plugin-eval/scripts/noise-report.py "$results" >"$OUT/$name/noise.txt" 2>&1 || true + printf '\nValidity: %s\n' "$verdict" >>"$summary" + if [ -n "$bash_ran" ]; then printf '\n%s\n' "$bash_ran" >>"$summary"; fi + fi + cat "$summary" >>"$GITHUB_STEP_SUMMARY" + echo >>"$GITHUB_STEP_SUMMARY" + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + cat "$summary" + echo "::$token::" + # summarize.mjs escapes the command it writes; it prints outside + # the stop-commands block so the runner reads it. + # No file at all: summarize.mjs could not read the result (exit 2). + if [ -s "$annotation" ]; then + head -n 1 "$annotation" + elif [ ! -e "$annotation" ]; then + echo "::error title=Plugin eval did not run%3A ${name//[$'\r\n%:,']/_}::The result could not be read." + fi + done <<<"$SUITES" + exit "$failed" + + # Opt-in because the job log of this public repository is public. Prints, + # per run, at most 20 Bash calls whose tool_result has is_error true: the + # command cut to 300 characters, and the result text cut to its first 120 + # and last 2500 characters when longer than 2620 (the end holds the daemon + # log), each JSON-quoted onto one line. A second line, `key:`, holds the lines of the + # full result that match FATAL, Check failed, ERROR:, sandbox, permission or errno + # patterns (the cause line that a long crash stack pushes out of the tail), minus the + # known .bashrc noise, de-duplicated, at most 15, each cut to 300 characters; it is + # omitted when nothing matches. Nothing else from the trace is printed. Trace text is + # tool output, so workflow commands are off while it prints. + - name: Print Bash error excerpts + if: always() && github.event_name == 'workflow_dispatch' && inputs.debug-errors && steps.stage.outputs.go == 'true' && steps.judge.outputs.handoff != 'true' + env: + OUT: ${{ runner.temp }}/eval-results + run: | + token=$(openssl rand -hex 16) + echo "::stop-commands::$token" + for RESULTS in "$OUT"/*/results.json; do + if [ ! -f "$RESULTS" ]; then + echo "no results.json, so no traces to read" + continue + fi + jq -r '.cases[]? as $c | $c.arms // {} | to_entries[] | .key as $arm + | .value // [] | to_entries[] + | [($c.name // $c.dir // "case"), $arm, (.key + 1 | tostring), (.value.tracePath // "")] + | @tsv' "$RESULTS" | + while IFS=$'\t' read -r name arm run trace; do + echo "== $name, $arm arm, run $run" + if [ ! -f "$trace" ]; then + echo " trace not found" + continue + fi + jq -nrR ' + def tail_cut: if length > 2620 then .[:120] + " โ€ฆ " + .[-2500:] else . end; + def key_lines: + [split("\n")[] + | select(test("FATAL|Check failed|ERROR:|No usable sandbox|Permission denied|Operation not permitted|EACCES|EPERM|ENOENT|errno") + and (contains(".bashrc: Permission denied") | not)) + | .[:300]] + | reduce .[] as $l ([]; if index([$l]) then . else . + [$l] end) + | .[:15] | join("\n"); + [inputs | fromjson? | objects] as $m + | ([$m[] | select(.type == "assistant") | .message.content[]? | objects + | select(.type == "tool_use" and .name == "Bash" and (.id | type) == "string") + | {key: .id, value: (.input.command // "" | tostring)}] | from_entries) as $bash + | [$m[] | select(.type == "user") | .message.content[]? | objects + | select(.type == "tool_result" and .is_error == true) + | (.tool_use_id // "" | tostring) as $id + | select($bash | has($id)) + | {cmd: $bash[$id], + out: (.content | if type == "array" then map(objects | select(.type == "text") | .text | tostring) | join("\n") else tostring end)}] + | " \(length) Bash error(s), showing up to 20", + (.[:20][] | " $ \(.cmd[:300] | tojson)\n -> \(.out | tail_cut | tojson)\(.out | key_lines | if . == "" then "" else "\n key: \(tojson)" end)") + ' "$trace" || echo " could not read the trace" + done || echo "could not read $RESULTS" + done + echo "::$token::" + + # Only a dispatched run, whose cases merged through review, uploads its + # results: a pull request's prompts could steer model output into a + # public artifact. + - name: Scan the upload for credentials + id: scan-upload + if: >- + always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/eval-results/ + + - name: Upload the eval results + if: >- + (always() && steps.stage.outputs.go == 'true' && github.event_name == 'workflow_dispatch' && steps.judge.outputs.handoff != 'true') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-browser-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/eval-results/ + if-no-files-found: warn + retention-days: 3 + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-browser-dispatch + lane-id: eval-browser-dispatch + credential: FALLBACK_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.eval-subscription.outputs.handoff-reason || 'no-key' }} + - name: Log the API budget at the end if: always() continue-on-error: true diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml index b251d86bc6c..244615f7698 100644 --- a/.github/workflows/eval-hill-climb-pass.yml +++ b/.github/workflows/eval-hill-climb-pass.yml @@ -14,6 +14,18 @@ name: eval-hill-climb-pass # tool is granted beyond the read-only set. Console output, which can carry # model text, goes to a file that nothing uploads. # +# The pass runs on the maintenance role's credential chain, one credential +# per job: `pass` on MAINTENANCE_CLAUDE_API_KEY, then `pass-subscription` on +# MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN, then `pass-fallback` on +# FALLBACK_CLAUDE_CODE_OAUTH_TOKEN. A leg hands off on the stops run-claude-cli +# hands off on (eval-handoff.mjs reads the probe's stream and every errored +# run's kept trace with run-claude-cli's own classifier): the key leg on a +# credit stop or a 401 or 402, the subscription leg, when the fallback token +# is set, also on a limit stop; an empty credential hands off too. A leg that +# hands off uploads nothing, so the pass's outputs are the one leg that +# finished. The gate job skips the key after a recent credit stop +# (ci-workflows check-key-credit). +# # The pass uploads only what eval-browser-dispatch.yml uploads, its result # (per-case scores and grader verdicts), with summarize.mjs's summary (its # health lines read from the kept traces) and run-validity.py's bash ran and @@ -60,21 +72,72 @@ on: type: string default: "" secrets: + MAINTENANCE_CLAUDE_API_KEY: + description: The maintenance role's Console API key, the key leg's credential. + required: false MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: - description: The maintenance role's subscription token. - required: true + description: The maintenance role's subscription token, the subscription leg's credential. + required: false + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: + description: The central fallback subscription token, the fallback leg's credential. + required: false outputs: artifact-id: - description: The pass artifact's id. - value: ${{ jobs.pass.outputs.artifact-id }} + description: The pass artifact's id, from the leg that finished the pass. + value: ${{ jobs.pass-fallback.outputs.artifact-id || jobs.pass-subscription.outputs.artifact-id || jobs.pass.outputs.artifact-id }} manifest-sha256: - description: SHA-256 of the artifact's manifest.sha256. - value: ${{ jobs.pass.outputs.manifest-sha256 }} + description: SHA-256 of that artifact's manifest.sha256. + value: ${{ jobs.pass-fallback.outputs.manifest-sha256 || jobs.pass-subscription.outputs.manifest-sha256 || jobs.pass.outputs.manifest-sha256 }} permissions: {} jobs: + # The key's credit gate (ci-workflows check-key-credit, gate.cjs: one + # artifact-list call for the marker the key leg uploads after a credit + # stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour unset) + # starts the pass on the subscription, with no refused key attempt first. + # No secret, no model, no checkout; every failure runs the key (skip-key + # empty). Probing is off. + gate: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: pass + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" + + # The key leg: the pass on the maintenance role's API key alone. A credit + # stop, or a 401 or 402, in the probe or in any run hands the pass to + # pass-subscription, and this leg uploads nothing. An empty key, or one + # the gate skipped, runs nothing here. pass: + needs: gate + if: ${{ !cancelled() && needs.gate.result == 'success' }} runs-on: ubuntu-24.04 timeout-minutes: 60 cache-mode: read @@ -83,6 +146,10 @@ jobs: outputs: artifact-id: ${{ steps.upload.outputs.artifact-id }} manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} env: npm_config_audit: "false" npm_config_fetch_timeout: "60000" @@ -116,18 +183,25 @@ jobs: echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" exit 1 - # Only whether the secret is set leaves this step, never its value. + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. - name: Check the credential + id: credential env: - PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.gate.outputs.skip-key }} run: | - if [ "$PRESENT" != true ]; then - echo "::error::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty, so no case runs" - exit 1 + run=$PRESENT + if [ "$SKIP_KEY" = true ]; then + run=false + echo "::notice::the credit gate skipped MAINTENANCE_CLAUDE_API_KEY; pass-subscription runs the pass" + elif [ "$PRESENT" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; pass-subscription runs the pass" fi + { echo "present=$PRESENT"; echo "run=$run"; } >>"$GITHUB_OUTPUT" - name: Download the edit - if: inputs.pass == 'b' + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: artifact-ids: ${{ inputs.edit-artifact-id }} @@ -136,7 +210,7 @@ jobs: # The patch the scan job checked, byte for byte, applied with git's own # path checks; nothing else of the artifact is read. - name: Apply the edit - if: inputs.pass == 'b' + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' env: PATCH: ${{ runner.temp }}/edit/edit.patch WANT: ${{ inputs.edit-sha256 }} @@ -156,6 +230,7 @@ jobs: # validate-cases.py makes no model call; the suite is then copied without # its judged cases. - name: Stage the suite + if: steps.credential.outputs.run == 'true' env: DIR: .base/plugins/${{ inputs.plugin }} run: | @@ -169,6 +244,7 @@ jobs: fi - name: Create the tool directories + if: steps.credential.outputs.run == 'true' run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools # eval-browser-dispatch.yml's pull request policy, for the same reason: @@ -176,6 +252,7 @@ jobs: # runs model-written skill text. - name: Deny the eval's reads of process and runner state id: deny + if: steps.credential.outputs.run == 'true' run: | runner_root="" pid=$$ while [ "${pid:-0}" -gt 1 ]; do @@ -210,35 +287,74 @@ jobs: echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" - name: Isolate the runner + if: steps.credential.outputs.run == 'true' uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] with: install-subprocess-isolation: "true" - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' timeout-minutes: 5 run: | cp .base/package.json .base/package-lock.json /opt/tools/ npm ci --prefix /opt/tools echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + # eval-browser-dispatch.yml's probe: the job stops unless the result # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } out=$(mktemp) trap 'rm -f "$out"' EXIT + status=0 claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ - --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format json >"$out" 2>/dev/null || true - if grep -qF "$CLAUDE_CODE_OAUTH_TOKEN" "$out"; then + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi denied() { - jq -e --arg t "$1" --arg p "$2" 'any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } } denied Read /proc/self/environ @@ -249,14 +365,20 @@ jobs: # Identical flags on every pass. TMPDIR sits outside /tmp, as in # eval-browser-dispatch.yml. - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' env: - CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + LANE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" DIR: .base/plugins/${{ inputs.plugin }} TAGS: ${{ inputs.tags }} CASE: ${{ inputs.case }} OUT: ${{ runner.temp }}/pass run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi export TMPDIR="$HOME/.ev" mkdir -p "$TMPDIR" "$OUT" args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 @@ -272,12 +394,37 @@ jobs: >"$RUNNER_TEMP/console.log" 2>&1 || status=$? echo "claude plugin eval exited $status" + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: key + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + # The summary (health lines from the kept traces) and the validity # verdict, read here because the traces stay on this runner. Only the # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's - # detail lines can quote a trace. + # detail lines can quote a trace. A leg that handed off reads nothing. - name: Read the pass id: read + if: steps.judge.outputs.handoff == 'false' env: OUT: ${{ runner.temp }}/pass PLUGIN: ${{ inputs.plugin }} @@ -302,16 +449,770 @@ jobs: - name: Scan the upload for credentials id: scan-upload + if: steps.read.outcome == 'success' uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] with: path: ${{ runner.temp }}/upload/ - name: Upload the pass id: upload - if: steps.scan-upload.outputs.clean == 'true' + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.run_id }} + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} path: ${{ runner.temp }}/upload/ retention-days: 1 if-no-files-found: error + + # Read by the next runs' credit gate: the artifact its `marker` output + # names (check-key-credit gate.cjs markerName), uploaded only after the + # key leg's credit-stop handoff. The name carries the caller workflow's + # file, since a called workflow's github context is the caller's, and is + # written out so its retention row claims it. pass-a and pass-a2 can both + # upload it in one run, so a failed upload never fails the job. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.gate.outputs.marker == 'key-credit-stop-maintenance-measure-skill-edit.yml-pass' && steps.judge.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + id: credit-marker-upload + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-maintenance-measure-skill-edit.yml-pass + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_API_KEY + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: "" + + # The subscription leg: the key leg's steps on a fresh runner, on the + # maintenance role's subscription token alone, with no credit-stop marker. + # With FALLBACK_CLAUDE_CODE_OAUTH_TOKEN set, a limit stop, a credit stop, + # or a 401 or 402 hands off to pass-fallback, and so does an empty token. + # hill-climb-workflow.test.mjs holds its steps equal to the key leg's but + # for the credential and the leg record's fallback reason. + pass-subscription: + needs: [gate, pass] + if: >- + ${{ !cancelled() && needs.gate.result == 'success' + && (needs.pass.outputs.handoff == 'true' || needs.pass.outputs.credential-present == 'false' + || needs.gate.outputs.skip-key == 'true') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the default branch; nothing in this job writes + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Check out the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.base-sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} + FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + if [ "$FALLBACK_PRESENT" != true ]; then + echo "::error::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are both empty, so no case runs" + exit 1 + fi + echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; pass-fallback runs the pass" + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" + + - name: Download the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ inputs.edit-artifact-id }} + path: ${{ runner.temp }}/edit + + # The patch the scan job checked, byte for byte, applied with git's own + # path checks; nothing else of the artifact is read. + - name: Apply the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + env: + PATCH: ${{ runner.temp }}/edit/edit.patch + WANT: ${{ inputs.edit-sha256 }} + run: | + if [ -L "$PATCH" ] || [ ! -f "$PATCH" ]; then + echo "::error::the edit artifact holds no edit.patch" + exit 1 + fi + sum=$(sha256sum <"$PATCH") + if ! [[ $WANT =~ ^[0-9a-f]{64}$ ]] || [ "${sum%% *}" != "$WANT" ]; then + echo "::error::edit.patch does not match the SHA-256 the scan job recorded" + exit 1 + fi + git -C .base apply --check "$PATCH" + git -C .base apply "$PATCH" + + # validate-cases.py makes no model call; the suite is then copied without + # its judged cases. + - name: Stage the suite + if: steps.credential.outputs.run == 'true' + env: + DIR: .base/plugins/${{ inputs.plugin }} + run: | + eval_dir=$(jq -r '.experimental.evals // "evals"' "$DIR/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$DIR/$eval_dir" + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$DIR" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::error::the suite has no case with deterministic graders only" + exit 1 + fi + + - name: Create the tool directories + if: steps.credential.outputs.run == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools + + # eval-browser-dispatch.yml's pull request policy, for the same reason: + # the eval's Read, Glob and Grep run without the Bash sandbox, and pass b + # runs model-written skill text. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.credential.outputs.run == 'true' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + + - name: Isolate the runner + if: steps.credential.outputs.run == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' + timeout-minutes: 5 + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # eval-browser-dispatch.yml's probe: the job stops unless the result + # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. + - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + + # Identical flags on every pass. TMPDIR sits outside /tmp, as in + # eval-browser-dispatch.yml. + - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + DIR: .base/plugins/${{ inputs.plugin }} + TAGS: ${{ inputs.tags }} + CASE: ${{ inputs.case }} + OUT: ${{ runner.temp }}/pass + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" "$OUT" + args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 + --max-cost-usd 5 --runs 3 --scaffold --keep-temp) + read -ra tags <<<"$TAGS" + for value in "${tags[@]}" "$CASE"; do + case "$value" in -*) echo "::error::a case or tag must not start with -"; exit 1 ;; esac + done + if [ "${#tags[@]}" -gt 0 ]; then args+=(--tag "${tags[@]}"); fi + if [ -n "$CASE" ]; then args+=(--case "$CASE"); fi + status=0 + claude plugin eval "$DIR" --eval-dir evals-deterministic --output-dir "$OUT/run" --json "$OUT/results.json" "${args[@]}" \ + >"$RUNNER_TEMP/console.log" 2>&1 || status=$? + echo "claude plugin eval exited $status" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: subscription + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The summary (health lines from the kept traces) and the validity + # verdict, read here because the traces stay on this runner. Only the + # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's + # detail lines can quote a trace. A leg that handed off reads nothing. + - name: Read the pass + id: read + if: steps.judge.outputs.handoff == 'false' + env: + OUT: ${{ runner.temp }}/pass + PLUGIN: ${{ inputs.plugin }} + PASS: ${{ inputs.pass }} + STAGE: ${{ runner.temp }}/upload + run: | + mkdir -p "$STAGE" + if [ ! -f "$OUT/results.json" ]; then + echo "::error::pass $PASS wrote no result" + exit 1 + fi + cp "$OUT/results.json" "$STAGE/results.json" + node .base/.github/scripts/plugin-eval/summarize.mjs "$OUT/results.json" --plugin "$PLUGIN" >"$STAGE/summary.md" || true + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$OUT/results.json" --runs 3 --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$(printf '%s\n' "$validity" | tail -n 1)" >"$STAGE/validity.txt" + cat "$STAGE/validity.txt" + sums=$(cd "$STAGE" && find . -type f -print0 | sort -z | xargs -0 sha256sum) + printf '%s\n' "$sums" >"$STAGE/manifest.sha256" + sum=$(sha256sum <"$STAGE/manifest.sha256") + echo "manifest-sha256=${sum%% *}" >>"$GITHUB_OUTPUT" + + - name: Scan the upload for credentials + id: scan-upload + if: steps.read.outcome == 'success' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/upload/ + + - name: Upload the pass + id: upload + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} + path: ${{ runner.temp }}/upload/ + retention-days: 1 + if-no-files-found: error + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.pass.outputs.handoff-reason || needs.gate.outputs.skip-key == 'true' && 'credit-stop' || needs.pass.outputs.credential-present == 'false' && 'no-key' || '' }} + + # The fallback leg: the same steps on the central fallback token, after + # the subscription leg handed off or found its token empty, and only when + # the fallback token is set. It is the last leg and never hands off. + pass-fallback: + needs: [gate, pass-subscription] + if: >- + ${{ !cancelled() && needs.pass-subscription.outputs.fallback-ready == 'true' + && (needs.pass-subscription.outputs.handoff == 'true' || needs.pass-subscription.outputs.credential-present == 'false') }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + cache-mode: read + permissions: + contents: read # check out the default branch; nothing in this job writes + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + manifest-sha256: ${{ steps.read.outputs.manifest-sha256 }} + handoff: ${{ steps.judge.outputs.handoff }} + handoff-reason: ${{ steps.judge.outputs.handoff-reason }} + credential-present: ${{ steps.credential.outputs.present }} + fallback-ready: ${{ steps.credential.outputs.fallback-ready }} + env: + npm_config_audit: "false" + npm_config_fetch_timeout: "60000" + npm_config_fund: "false" + npm_config_globalconfig: ${{ github.workspace }}/.base/.github/npm-fetch.npmrc + steps: + - name: Check out the default branch + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.base-sha }} + path: .base + persist-credentials: false + + - name: Set up Node + uses: ./.base/.github/actions/setup-pinned-node # zizmor: ignore[self-repository] + with: + node-version-file: .base/.node-version + + - name: Check the kill switch + id: kill + uses: ./.base/.github/actions/check-kill-switch # zizmor: ignore[self-repository] + with: + value: ${{ vars.CLAUDE_LANES_DISABLED }} + repo-value: ${{ vars.CLAUDE_LANES_REPO_DISABLED }} + + - name: Stop when the kill switch is on + if: steps.kill.outputs.proceed != 'true' + env: + REASON: ${{ steps.kill.outputs.reason }} + run: | + echo "::error::the lanes kill switch stops this run (reason: ${REASON:-unreadable})" + exit 1 + + # Only whether each secret is set leaves this step, never its value. + # `run` is whether this leg runs the pass. + - name: Check the credential + id: credential + env: + PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} + run: | + if [ "$PRESENT" != true ]; then + echo "::error::FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is empty, so no case runs" + exit 1 + fi + { echo "present=$PRESENT"; echo "run=$PRESENT"; } >>"$GITHUB_OUTPUT" + + - name: Download the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ inputs.edit-artifact-id }} + path: ${{ runner.temp }}/edit + + # The patch the scan job checked, byte for byte, applied with git's own + # path checks; nothing else of the artifact is read. + - name: Apply the edit + if: steps.credential.outputs.run == 'true' && inputs.pass == 'b' + env: + PATCH: ${{ runner.temp }}/edit/edit.patch + WANT: ${{ inputs.edit-sha256 }} + run: | + if [ -L "$PATCH" ] || [ ! -f "$PATCH" ]; then + echo "::error::the edit artifact holds no edit.patch" + exit 1 + fi + sum=$(sha256sum <"$PATCH") + if ! [[ $WANT =~ ^[0-9a-f]{64}$ ]] || [ "${sum%% *}" != "$WANT" ]; then + echo "::error::edit.patch does not match the SHA-256 the scan job recorded" + exit 1 + fi + git -C .base apply --check "$PATCH" + git -C .base apply "$PATCH" + + # validate-cases.py makes no model call; the suite is then copied without + # its judged cases. + - name: Stage the suite + if: steps.credential.outputs.run == 'true' + env: + DIR: .base/plugins/${{ inputs.plugin }} + run: | + eval_dir=$(jq -r '.experimental.evals // "evals"' "$DIR/.claude-plugin/plugin.json" 2>/dev/null || echo evals) + python3 .base/plugins/evals/skills/validate/scripts/validate-cases.py "$DIR/$eval_dir" + split=$(node .base/.github/scripts/plugin-eval/deterministic-suite.mjs "$DIR" evals-deterministic) + echo "$split" + if [ "$(jq '.kept | length' <<<"$split")" -eq 0 ]; then + echo "::error::the suite has no case with deterministic graders only" + exit 1 + fi + + - name: Create the tool directories + if: steps.credential.outputs.run == 'true' + run: sudo install -d -o "$(id -u)" -g "$(id -g)" /opt/tools + + # eval-browser-dispatch.yml's pull request policy, for the same reason: + # the eval's Read, Glob and Grep run without the Bash sandbox, and pass b + # runs model-written skill text. + - name: Deny the eval's reads of process and runner state + id: deny + if: steps.credential.outputs.run == 'true' + run: | + runner_root="" pid=$$ + while [ "${pid:-0}" -gt 1 ]; do + exe=$(readlink -f "/proc/$pid/exe" 2>/dev/null || true) + case "$exe" in */bin/Runner.Worker) runner_root=${exe%/bin/Runner.Worker}; break ;; esac + pid=$(awk '/^PPid:/ {print $2}' "/proc/$pid/status" 2>/dev/null) + done + if [ -z "$runner_root" ]; then + echo "::error::no Runner.Worker process above this step, so the runner's directory cannot be denied" + exit 1 + fi + work=$(dirname "$RUNNER_WORKSPACE") + policy=$(mktemp) + jq -n --arg runner "Read(/$runner_root/**)" --arg temp "Read(/$work/_temp/**)" --arg actions "Read(/$work/_actions/**)" \ + '{permissions: {deny: [ + "Read(//proc/**)", + "Read(//sys/**)", + "Read(//etc/claude-code/**)", + $runner, + "Read(//home/runner/runners/**)", + "Read(//home/runner/actions-runner/**)", + $temp, + $actions, + "Read(//home/runner/.config/**)", + "Read(//home/runner/.ssh/**)", + "Read(//home/runner/.docker/**)", + "Read(~/.claude/**)", + "Read(~/.claude.json)" + ]}}' >"$policy" + sudo install -D -m 0644 -o root -g root "$policy" /etc/claude-code/managed-settings.json + jq -e '.permissions.deny | index("Read(//proc/**)")' /etc/claude-code/managed-settings.json >/dev/null + echo "runner-root=$runner_root" >>"$GITHUB_OUTPUT" + + - name: Isolate the runner + if: steps.credential.outputs.run == 'true' + uses: ./.base/.github/actions/setup-runner-isolation # zizmor: ignore[self-repository] + with: + install-subprocess-isolation: "true" + + - name: Install the locked root toolchain + if: steps.credential.outputs.run == 'true' + timeout-minutes: 5 + run: | + cp .base/package.json .base/package-lock.json /opt/tools/ + npm ci --prefix /opt/tools + echo "/opt/tools/node_modules/.bin" >> "$GITHUB_PATH" + + # run-claude-cli's classifier, at the pin every lane runs it, for + # eval-handoff.mjs: lane.cjs and the two files it loads, nothing run here. + - name: Check out the launcher's classifier + if: steps.credential.outputs.run == 'true' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: melodic-software/ci-workflows + ref: b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + path: .ci-workflows + persist-credentials: false + sparse-checkout: | + .github/actions/run-claude-cli/lane.cjs + .github/actions/report-lane-outcome/classify.cjs + .github/scripts/resolve-npm-tool-version.cjs + sparse-checkout-cone-mode: false + + # eval-browser-dispatch.yml's probe: the job stops unless the result + # records each of the four reads as denied, or if it holds the credential. + # A probe that the credential's own stop ended hands off instead + # (eval-handoff.mjs), so the next leg runs the pass. The stream stays in a + # file that is never printed and is removed after the checks. + - name: Probe that the policy denies those reads + id: probe + if: steps.credential.outputs.run == 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + RUNNER_ROOT: ${{ steps.deny.outputs.runner-root }} + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + credential=${ANTHROPIC_API_KEY:-${CLAUDE_CODE_OAUTH_TOKEN:-}} + [ -n "$credential" ] || { echo "::error::the leg's credential is empty"; exit 1; } + out=$(mktemp) + trap 'rm -f "$out"' EXIT + status=0 + claude -p "Make exactly these four tool calls, in order, and no other: the Read tool on /proc/self/environ; the Read tool on $RUNNER_ROOT/.runner; the Grep tool with pattern TOKEN and path /proc/self/environ; the Glob tool with pattern environ and path /proc/self." \ + --model claude-sonnet-5-5 --allowedTools Read Grep Glob --max-turns 8 --output-format stream-json --verbose >"$out" 2>/dev/null || status=$? + if grep -qF "$credential" "$out"; then + echo "::error::the probe's result holds the credential, so no case runs" + exit 1 + fi + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" + if grep -qx 'handoff=true' <<<"$verdict"; then + echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" + exit 0 + fi + denied() { + jq -e -s --arg t "$1" --arg p "$2" 'last(.[] | select(.type == "result")) | any(.permission_denials[]?; .tool_name == $t and ((.tool_input.file_path // .tool_input.path // "") | startswith($p)))' "$out" >/dev/null 2>&1 || + { echo "::error::the managed policy did not record a denied $1 of $2, so no case runs"; exit 1; } + } + denied Read /proc/self/environ + denied Read "$RUNNER_ROOT/.runner" + denied Grep /proc/self + denied Glob /proc/self + + # Identical flags on every pass. TMPDIR sits outside /tmp, as in + # eval-browser-dispatch.yml. + - name: Run the pass + id: eval + if: steps.probe.outcome == 'success' && steps.probe.outputs.handoff != 'true' + env: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + CLAUDE_CODE_SUBPROCESS_ENV_SCRUB: "1" + DIR: .base/plugins/${{ inputs.plugin }} + TAGS: ${{ inputs.tags }} + CASE: ${{ inputs.case }} + OUT: ${{ runner.temp }}/pass + run: | + if [ -n "${LANE_API_KEY:-}" ]; then + export ANTHROPIC_API_KEY="$LANE_API_KEY" + unset LANE_API_KEY + fi + export TMPDIR="$HOME/.ev" + mkdir -p "$TMPDIR" "$OUT" + args=(--trust-plugin --no-publish --model claude-sonnet-5-5 --judge-model claude-opus-5-5 + --max-cost-usd 5 --runs 3 --scaffold --keep-temp) + read -ra tags <<<"$TAGS" + for value in "${tags[@]}" "$CASE"; do + case "$value" in -*) echo "::error::a case or tag must not start with -"; exit 1 ;; esac + done + if [ "${#tags[@]}" -gt 0 ]; then args+=(--tag "${tags[@]}"); fi + if [ -n "$CASE" ]; then args+=(--case "$CASE"); fi + status=0 + claude plugin eval "$DIR" --eval-dir evals-deterministic --output-dir "$OUT/run" --json "$OUT/results.json" "${args[@]}" \ + >"$RUNNER_TEMP/console.log" 2>&1 || status=$? + echo "claude plugin eval exited $status" + + # Whether the credential's own stop ended any run, read from each errored + # run's kept trace by run-claude-cli's classifier; nothing here prints a + # trace. A probe's handoff stands. The leg's execution file carries only + # the suite's durationSeconds and costUsd, for the lane record. + - name: Judge the leg + id: judge + if: always() && steps.probe.outcome == 'success' + env: + LEG: fallback + FALLBACK_READY: ${{ steps.credential.outputs.fallback-ready }} + CI_WORKFLOWS: ${{ github.workspace }}/.ci-workflows + RESULTS: ${{ runner.temp }}/pass/results.json + LEG_RESULT: ${{ runner.temp }}/leg-result.json + PROBE_HANDOFF: ${{ steps.probe.outputs.handoff }} + PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} + PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} + run: | + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + if [ "$PROBE_HANDOFF" = true ]; then + verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") + fi + printf '%s\n' "$verdict" | tee -a "$GITHUB_OUTPUT" + echo "leg-result=$LEG_RESULT" >>"$GITHUB_OUTPUT" + + # The summary (health lines from the kept traces) and the validity + # verdict, read here because the traces stay on this runner. Only the + # count-only bash ran line (the line just before the verdict, in its exact format) and the verdict line leave it: run-validity.py's + # detail lines can quote a trace. A leg that handed off reads nothing. + - name: Read the pass + id: read + if: steps.judge.outputs.handoff == 'false' + env: + OUT: ${{ runner.temp }}/pass + PLUGIN: ${{ inputs.plugin }} + PASS: ${{ inputs.pass }} + STAGE: ${{ runner.temp }}/upload + run: | + mkdir -p "$STAGE" + if [ ! -f "$OUT/results.json" ]; then + echo "::error::pass $PASS wrote no result" + exit 1 + fi + cp "$OUT/results.json" "$STAGE/results.json" + node .base/.github/scripts/plugin-eval/summarize.mjs "$OUT/results.json" --plugin "$PLUGIN" >"$STAGE/summary.md" || true + validity=$(python3 .base/plugins/evals/skills/plugin-eval/scripts/run-validity.py "$OUT/results.json" --runs 3 --scaffolded) || true + bash_ran=$(printf '%s\n' "$validity" | tail -n 2 | head -n 1 | grep -xE 'bash ran: (n/a|[a-z-]+ (n/a|[0-9]+ of [0-9]+)(, [a-z-]+ (n/a|[0-9]+ of [0-9]+))*(, unknown [0-9]+)?)' || true) + printf '%s\n' ${bash_ran:+"$bash_ran"} "$(printf '%s\n' "$validity" | tail -n 1)" >"$STAGE/validity.txt" + cat "$STAGE/validity.txt" + sums=$(cd "$STAGE" && find . -type f -print0 | sort -z | xargs -0 sha256sum) + printf '%s\n' "$sums" >"$STAGE/manifest.sha256" + sum=$(sha256sum <"$STAGE/manifest.sha256") + echo "manifest-sha256=${sum%% *}" >>"$GITHUB_OUTPUT" + + - name: Scan the upload for credentials + id: scan-upload + if: steps.read.outcome == 'success' + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/upload/ + + - name: Upload the pass + id: upload + if: (steps.read.outcome == 'success') && steps.scan-upload.outputs.clean == 'true' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: eval-hill-climb-pass-${{ inputs.pass }}-${{ github.job }}-${{ github.run_id }} + path: ${{ runner.temp }}/upload/ + retention-days: 1 + if-no-files-found: error + + # The leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs, one record per leg. + - name: Record the leg + if: always() && steps.judge.outcome == 'success' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.judge.outputs.outcome }} + execution-file: ${{ steps.judge.outputs.leg-result }} + lane: eval-hill-climb-pass + lane-id: eval-hill-climb-pass + credential: FALLBACK_CLAUDE_CODE_OAUTH_TOKEN + handoff: ${{ steps.judge.outputs.handoff }} + known-failure-class: ${{ steps.judge.outputs.lane-class }} + fallback-reason: ${{ needs.pass-subscription.outputs.handoff-reason || 'no-key' }} diff --git a/.github/workflows/maintenance-measure-skill-edit.yml b/.github/workflows/maintenance-measure-skill-edit.yml index 6adf500dfc8..f65a5b5fea0 100644 --- a/.github/workflows/maintenance-measure-skill-edit.yml +++ b/.github/workflows/maintenance-measure-skill-edit.yml @@ -10,9 +10,10 @@ name: maintenance-measure-skill-edit # keeps a case once judged graders are dropped. # pass-a, the unchanged skill twice (eval-hill-climb-pass.yml), each on its # pass-a2 own runner, 3 runs per case per arm under --max-cost-usd 5. -# propose win-test.py's baseline gates, then one run-claude-cli leg -# (--restricted, read-only GITHUB_TOKEN, no write token, read-only -# tools) that returns one edit as structured output. +# propose win-test.py's baseline gates, then run-claude-cli on the key, +# subscription and fallback legs in turn (--restricted, read-only +# GITHUB_TOKEN, no write token, read-only tools), the first that +# finishes returning one edit as structured output. # scan no model, no secret: applies the edit to a fresh checkout and # refuses it outside plugins/

/skills//, under evals/, or # as select-plugins.mjs refuses a pull request's change. @@ -39,8 +40,10 @@ name: maintenance-measure-skill-edit # publish hold id-token, each to mint one lanes App token from the broker, # which refuses this caller on any event but workflow_dispatch and any actor # or re-runner but a listed human. No job holds a merge right, checks or -# statuses write, or workflows. The credential is the maintenance role's -# subscription token, as in eval-browser-dispatch.yml. How to stop it: set the +# statuses write, or workflows. Every pass and the proposal run on the +# maintenance role's chain, as in eval-browser-dispatch.yml: its API key, then +# its subscription token, then the central fallback token, one per job or +# step, each after the one before hands off. How to stop it: set the # organization or repository lanes kill-switch variable to anything but # "false"; every job that spends or writes checks it first, and the broker # reads it live. @@ -212,6 +215,7 @@ jobs: if: needs.gate.outputs.go == 'true' permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -220,13 +224,16 @@ jobs: pass: a base-sha: ${{ needs.gate.outputs.base-sha }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} pass-a2: needs: gate if: needs.gate.outputs.go == 'true' permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -235,14 +242,58 @@ jobs: pass: a2 base-sha: ${{ needs.gate.outputs.base-sha }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} + + # The proposer's key-credit gate (ci-workflows check-key-credit, gate.cjs: + # one artifact-list call for the marker propose uploads after a key leg's + # credit stop). A credit stop inside vars.KEY_CREDIT_WINDOW_HOURS (1 hour + # unset) starts the proposal on the subscription, with no refused key + # attempt first. No secret, no model, no checkout; every failure runs the + # key (skip-key empty). Probing is off. + propose-key-credit: + needs: [gate, pass-a, pass-a2] + if: needs.pass-a.result == 'success' && needs.pass-a2.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # list this repository's credit-stop markers + outputs: + skip-key: ${{ steps.credit.outputs.skip-key }} + marker: ${{ steps.credit.outputs.marker }} + steps: + - name: Read the key's credit history + id: credit + continue-on-error: true + timeout-minutes: 2 + uses: melodic-software/ci-workflows/.github/actions/check-key-credit@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + with: + key-job: propose + window-hours: ${{ vars.KEY_CREDIT_WINDOW_HOURS || '1' }} + probe-minutes: "0" + + # The skipped key leg's lane record: outcome skip, class credit-stop, + # handed off, so leg counts before and after the gate compare. + - name: Report the skipped key leg + if: steps.credit.outputs.skip-key == 'true' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: skipped + lane: maintenance-measure-skill-edit + lane-id: maintenance-measure-skill-edit + credential: MAINTENANCE_CLAUDE_API_KEY + known-failure-class: credit-stop + handoff: "true" - # The model, with the subscription token, a read-only GITHUB_TOKEN and + # The model, with one maintenance credential per leg (the API key, then the + # subscription token, then the fallback token), a read-only GITHUB_TOKEN and # nothing else. run-claude-cli runs the CLI with --restricted, which confines - # the file tools to the workspace; the leg gets Read, Grep and Glob only, so + # the file tools to the workspace; each leg gets Read, Grep and Glob only, so # its one result is the structured output, which the scan job checks. propose: - needs: [gate, pass-a, pass-a2] + needs: [gate, pass-a, pass-a2, propose-key-credit] if: needs.pass-a.result == 'success' && needs.pass-a2.result == 'success' runs-on: ubuntu-24.04 timeout-minutes: 40 @@ -385,22 +436,44 @@ jobs: printf '%s\n' "$prompt" >"$PROMPT_FILE" printf '%s\n' '{"type":"object","properties":{"file":{"type":"string","minLength":1,"maxLength":200},"old_string":{"type":"string","minLength":1,"maxLength":20000},"new_string":{"type":"string","maxLength":20000},"rationale":{"type":"string","maxLength":4000},"findings":{"type":"array","maxItems":10,"items":{"type":"string","maxLength":500}}},"required":["file","old_string","new_string","rationale","findings"],"additionalProperties":false}' >"$SCHEMA_FILE" - # Which legs can run, never a token's value. The maintenance role holds - # no API key (#13586): the subscription leg runs first, and the central - # fallback token's leg after it hands off (a limit stop, a credit stop, - # or a 401 or 402) or when its token is empty. + # Which legs can run, never a secret's value. The key runs unless it is + # empty or propose-key-credit skipped it; key-reason says why the + # proposal starts on the subscription. + - name: Check the API key + id: key + if: steps.prompt.outcome == 'success' + env: + KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY != '' }} + SKIP_KEY: ${{ needs.propose-key-credit.outputs.skip-key }} + run: | + key=$KEY + reason="" + if [ "$SKIP_KEY" = true ]; then + key=false + reason=credit-stop + elif [ "$KEY" != true ]; then + echo "::warning::MAINTENANCE_CLAUDE_API_KEY is empty; the subscription leg proposes" + reason=no-key + fi + { echo "key=$key"; echo "key-reason=$reason"; } >>"$GITHUB_OUTPUT" + + # The subscription leg runs after the key leg hands off (a credit stop, + # or a 401 or 402) or did not run, and the central fallback token's leg + # after the subscription leg hands off (a limit stop, a credit stop, or + # a 401 or 402) or finds its token empty. - name: Check the credentials id: credential if: steps.prompt.outcome == 'success' env: + KEY_USABLE: ${{ steps.key.outputs.key }} PRESENT: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN != '' }} FALLBACK_PRESENT: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN != '' }} run: | - if [ "$PRESENT" != true ]; then + if [ "$KEY_USABLE" != true ] && [ "$PRESENT" != true ]; then if [ "$FALLBACK_PRESENT" = true ]; then echo "::warning::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN is empty; the fallback leg proposes on FALLBACK_CLAUDE_CODE_OAUTH_TOKEN" else - echo "::notice::MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN and FALLBACK_CLAUDE_CODE_OAUTH_TOKEN are both empty; no leg runs" + echo "::notice::no maintenance credential and no FALLBACK_CLAUDE_CODE_OAUTH_TOKEN is usable; no leg runs" fi fi { echo "present=$PRESENT"; echo "fallback-ready=$FALLBACK_PRESENT"; } >>"$GITHUB_OUTPUT" @@ -415,9 +488,52 @@ jobs: with: install-subprocess-isolation: "true" + # The three legs hold one credential each and stay in one job: the model + # has Read, Grep and Glob only, no Bash (run-claude-cli's one-job rule). + - name: Propose the edit on the API key + id: run-key + if: steps.prompt.outcome == 'success' && steps.key.outputs.key == 'true' + continue-on-error: true + timeout-minutes: 30 + uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 + env: + CLAUDE_CODE_DISABLE_BACKGROUND_TASKS: "1" + with: + lane: maintenance-measure-skill-edit + role: maintenance + prompt-file: ${{ runner.temp }}/hill-climb-prompt.md + preamble: You run unattended as the eval hill-climb CI lane. No one can answer a question during this run, and you change nothing. + api-key: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} + model: ${{ steps.models.outputs.model }} + effort: ${{ steps.models.outputs.effort }} + max-turns: ${{ steps.models.outputs.max-turns }} + permission-mode: dontAsk + tools: | + Read + Grep + Glob + allowed-tools: | + Read(./hill-climb-data/**) + Read(./.base/plugins/**) + disallowed-tools: | + Bash + Edit + Write + NotebookEdit + WebFetch + WebSearch + Agent + Skill + Read(//proc/**) + mcp__* + json-schema-file: ${{ runner.temp }}/hill-climb-schema.json + leg-timeout-minutes: "25" + - name: Propose the edit id: run - if: steps.prompt.outcome == 'success' && steps.credential.outputs.present == 'true' + if: >- + steps.prompt.outcome == 'success' && steps.credential.outputs.present == 'true' + && (steps.run-key.outcome == 'skipped' || steps.run-key.outputs.handoff == 'true') continue-on-error: true timeout-minutes: 30 uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 @@ -459,7 +575,8 @@ jobs: id: run-fallback if: >- steps.prompt.outcome == 'success' && steps.credential.outputs.fallback-ready == 'true' - && (steps.run.outputs.handoff == 'true' || steps.credential.outputs.present != 'true') + && (steps.run.outputs.handoff == 'true' + || steps.credential.outputs.present != 'true' && (steps.run-key.outcome == 'skipped' || steps.run-key.outputs.handoff == 'true')) continue-on-error: true timeout-minutes: 30 uses: melodic-software/ci-workflows/.github/actions/run-claude-cli@b8bdd7fa2319b2457cbbef32c5348c394b9bc3f6 # b8bdd7f 2026-11-04 @@ -496,6 +613,15 @@ jobs: json-schema-file: ${{ runner.temp }}/hill-climb-schema.json leg-timeout-minutes: "25" + - name: Upload the key leg's sealed transcript + if: always() && steps.run-key.outputs.execution-file != '' + continue-on-error: true + uses: ./.base/.github/actions/upload-lane-transcript # zizmor: ignore[self-repository] + with: + execution-file: ${{ steps.run-key.outputs.execution-file }} + public-key: ${{ vars.LANE_TRANSCRIPT_PUBLIC_KEY }} + job-id: ${{ github.job }}-key + - name: Upload the sealed transcript if: always() && steps.run.outputs.execution-file != '' continue-on-error: true @@ -514,6 +640,58 @@ jobs: public-key: ${{ vars.LANE_TRANSCRIPT_PUBLIC_KEY }} job-id: ${{ github.job }}-fallback + # Read by the next runs' credit gate (propose-key-credit): the artifact + # its `marker` output names (check-key-credit gate.cjs markerName), + # uploaded only after the key leg's credit-stop handoff. The name is + # written out so its retention row claims it. + - name: Write the key's credit-stop marker + id: credit-marker + if: always() && needs.propose-key-credit.outputs.marker == 'key-credit-stop-maintenance-measure-skill-edit.yml-propose' && steps.run-key.outputs.handoff-reason == 'credit-stop' + continue-on-error: true + timeout-minutes: 1 + run: | + rm -rf "${RUNNER_TEMP:?}/key-credit-stop" + mkdir -p "$RUNNER_TEMP/key-credit-stop" + echo "$GITHUB_RUN_ID" >"$RUNNER_TEMP/key-credit-stop/run-id" + + - name: Scan the key's credit-stop marker for credentials + id: credit-marker-scan + if: always() && steps.credit-marker.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: ./.base/.github/actions/scan-upload # zizmor: ignore[self-repository] + with: + path: ${{ runner.temp }}/key-credit-stop/run-id + + - name: Upload the key's credit-stop marker + if: (always() && steps.credit-marker.outcome == 'success') && steps.credit-marker-scan.outputs.clean == 'true' + continue-on-error: true + timeout-minutes: 1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: key-credit-stop-maintenance-measure-skill-edit.yml-propose + path: ${{ runner.temp }}/key-credit-stop/run-id + if-no-files-found: error + retention-days: 1 + + # The key leg's lane-outcome record: its credential, outcome, handoff, + # fallback reason and durationMs. + - name: Record the key leg + if: always() && steps.run-key.outcome != 'skipped' && steps.run-key.outcome != '' + continue-on-error: true + uses: melodic-software/ci-workflows/.github/actions/report-lane-outcome@6b9c640bf209c0c7e76f1111c5d18c96707348a2 # 6b9c640 2026-11-02 + with: + outcome: ${{ steps.run-key.outcome }} + execution-file: ${{ steps.run-key.outputs.execution-file }} + log-file: ${{ steps.run-key.outputs.log-file }} + lane: maintenance-measure-skill-edit + lane-id: maintenance-measure-skill-edit + credential: ${{ steps.run-key.outputs.credential }} + handoff: ${{ steps.run-key.outputs.handoff }} + known-failure-class: ${{ steps.run-key.outputs.lane-class }} + max-turns: ${{ steps.models.outputs.max-turns }} + fallback-reason: "" + # The subscription leg's lane-outcome record: its credential, outcome, handoff, # fallback reason and durationMs. - name: Record the subscription leg @@ -530,7 +708,7 @@ jobs: handoff: ${{ steps.run.outputs.handoff }} known-failure-class: ${{ steps.run.outputs.lane-class }} max-turns: ${{ steps.models.outputs.max-turns }} - fallback-reason: "" + fallback-reason: ${{ steps.run-key.outputs.handoff-reason || steps.key.outputs.key-reason }} # The fallback leg's lane-outcome record: its credential, outcome, handoff, # fallback reason and durationMs. @@ -548,7 +726,7 @@ jobs: handoff: ${{ steps.run-fallback.outputs.handoff }} known-failure-class: ${{ steps.run-fallback.outputs.lane-class }} max-turns: ${{ steps.models.outputs.max-turns }} - fallback-reason: ${{ steps.run.outputs.handoff-reason || steps.credential.outputs.present != 'true' && 'no-key' || '' }} + fallback-reason: ${{ steps.run.outputs.handoff-reason || 'no-key' }} # Read from the launcher's file only after the leg's init message reports # dontAsk; the scan job validates it. @@ -557,10 +735,11 @@ jobs: if: >- steps.run-fallback.outcome == 'success' && steps.run-fallback.outputs.structured-output-file != '' || steps.run.outcome == 'success' && steps.run.outputs.structured-output-file != '' + || steps.run-key.outcome == 'success' && steps.run-key.outputs.structured-output-file != '' env: - EXECUTION_FILE: ${{ steps.run-fallback.outputs.execution-file || steps.run.outputs.execution-file }} + EXECUTION_FILE: ${{ steps.run-fallback.outputs.execution-file || steps.run.outputs.execution-file || steps.run-key.outputs.execution-file }} MODE: dontAsk - OUTPUT_FILE: ${{ steps.run-fallback.outputs.structured-output-file || steps.run.outputs.structured-output-file }} + OUTPUT_FILE: ${{ steps.run-fallback.outputs.structured-output-file || steps.run.outputs.structured-output-file || steps.run-key.outputs.structured-output-file }} run: | bash .base/.github/scripts/read-patch/check-permission-mode.sh mkdir -p "$RUNNER_TEMP/proposal" @@ -689,6 +868,7 @@ jobs: if: ${{ !cancelled() && needs.scan.outputs.artifact-id != '' }} permissions: contents: read + actions: read # the pass's gate job lists this repository's key markers uses: ./.github/workflows/eval-hill-climb-pass.yml # zizmor: ignore[self-repository] with: plugin: ${{ needs.gate.outputs.plugin }} @@ -699,7 +879,9 @@ jobs: edit-artifact-id: ${{ needs.scan.outputs.artifact-id }} edit-sha256: ${{ needs.scan.outputs.sha256 }} secrets: + MAINTENANCE_CLAUDE_API_KEY: ${{ secrets.MAINTENANCE_CLAUDE_API_KEY }} MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN }} + FALLBACK_CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.FALLBACK_CLAUDE_CODE_OAUTH_TOKEN }} # No model, no secret. Names the gate that stopped the run, with its # numbers; on a win, keeps the edit and each pass's results for 3 days.