diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 830d3a288e1..cd838bee3a4 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -9123,6 +9123,16 @@ for (const [file, gateId, [keyId, subId, fbId], marker] of [ assert.match(step.env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB, /^(?:"?1"?|\$\{\{ .+ \}\})$/, `${id} ${step.name} scrubs Bash subprocesses`); } assert.equal(steps.find((s) => s.id === "judge").env.LEG, leg, id); + // A leg whose credential is empty, or whose key the gate skipped, runs + // nothing past its credential check: every later step but the budget + // log has a condition, and the first ones read `run`. + const after = steps.slice(steps.findIndex((s) => s.id === "credential") + 1); + assert.deepEqual(after.filter((s) => s.if === undefined || s.if === "always()").map((s) => s.name).filter((n) => n !== "Log the API budget at the end"), [], id); + assert.match(after[0].if, /^steps\.credential\.outputs\.run == 'true'/, id); + // A classifier that cannot run stops the leg red, never as a handoff. + for (const step of steps.filter((s) => /plugin-eval\/eval-handoff\.mjs/.test(String(s.run ?? "")))) { + assert.match(step.run, /eval-handoff\.mjs\) \|\|\n\s+\{ echo "::error::the handoff classifier failed[^"]*"; exit 1; \}/, `${id} ${step.name}`); + } const checkout = steps.find((s) => s.with?.repository === "melodic-software/ci-workflows"); assert.equal(checkout.with.ref, CLASSIFIER_PIN, `${id} reads the classifier at the run-claude-cli pin`); assert.equal(checkout.with["persist-credentials"], false); diff --git a/.github/scripts/plugin-eval/eval-handoff.mjs b/.github/scripts/plugin-eval/eval-handoff.mjs index 4187905dd96..140b205533e 100755 --- a/.github/scripts/plugin-eval/eval-handoff.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.mjs @@ -25,7 +25,7 @@ // one result message carrying the suites' summed durationSeconds and costUsd, // nothing a run wrote). // Prints handoff, handoff-reason, lane-class and outcome as key=value lines. -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, isAbsolute, join } from "node:path"; @@ -103,13 +103,36 @@ function readJson(path) { } } +// The classifier's three files, copied under a folder of their own whose +// package.json ends Node's package-scope lookup there. Required in place, the +// lookup climbs past the sparse checkout to the workspace root, and a +// package.json there that does not parse stops the require (run 42138674762). +export const CLASSIFIER_FILES = [ + ".github/actions/run-claude-cli/lane.cjs", + ".github/actions/report-lane-outcome/classify.cjs", + ".github/scripts/resolve-npm-tool-version.cjs", +]; +export function loadLauncher(root) { + const dir = mkdtempSync(join(tmpdir(), "eval-handoff-classifier-")); + try { + writeFileSync(join(dir, "package.json"), '{"type":"commonjs"}\n'); + for (const path of CLASSIFIER_FILES) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + copyFileSync(join(root, path), join(dir, path)); + } + const require = createRequire(join(dir, CLASSIFIER_FILES[0])); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + return { judgeLeg, linesToMessages, isCreditStop }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function main(env) { const leg = LEGS[env.LEG]; if (!leg) throw new Error(`LEG must be key, subscription or fallback, not ${JSON.stringify(env.LEG)}`); - const root = env.CI_WORKFLOWS ?? ""; - const require = createRequire(join(root, ".github/actions/run-claude-cli/lane.cjs")); - const { judgeLeg, linesToMessages } = require("./lane.cjs"); - const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + const { judgeLeg, linesToMessages, isCreditStop } = loadLauncher(env.CI_WORKFLOWS ?? ""); const streams = []; if (env.PROBE && existsSync(env.PROBE)) { const exitCode = Number.parseInt(env.PROBE_EXIT ?? "", 10); diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs index c97127bd997..5c1cc7553a7 100644 --- a/.github/scripts/plugin-eval/eval-handoff.test.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -9,11 +9,10 @@ // records (CLI 2.1.302). import assert from "node:assert/strict"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { createRequire } from "node:module"; import { join } from "node:path"; import { test } from "node:test"; import { tempDir } from "../../../lib/test-tmp.mjs"; -import { erroredTraces, handoffReason, judgeLegStreams } from "./eval-handoff.mjs"; +import { CLASSIFIER_FILES, erroredTraces, handoffReason, judgeLegStreams, loadLauncher } from "./eval-handoff.mjs"; test("a key leg hands off on a credit stop or a 401 or 402, never a 403 or a limit stop", () => { assert.equal(handoffReason("api-key", "credit-stop", 400), "credit-stop"); @@ -66,7 +65,6 @@ test("the first stream that hands off decides, else the first stop class is kept // a pin bump that reshapes judgeLeg's return (failureClass, resets, the last // message's api_error_status) or isCreditStop fails here. The three paths are // the legs' sparse checkout. -const CLASSIFIER_FILES = [".github/actions/run-claude-cli/lane.cjs", ".github/actions/report-lane-outcome/classify.cjs", ".github/scripts/resolve-npm-tool-version.cjs"]; const workflowPins = ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"].flatMap((name) => [ ...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/repository: melodic-software\/ci-workflows\n\s+ref: ([0-9a-f]{40})/g), ].map((m) => m[1])); @@ -79,12 +77,50 @@ for (const path of CLASSIFIER_FILES) { mkdirSync(join(ROOT, path, ".."), { recursive: true }); writeFileSync(join(ROOT, path), await response.text()); } -const real = (() => { - const require = createRequire(join(ROOT, ".github/actions/run-claude-cli/lane.cjs")); - const { judgeLeg, linesToMessages } = require("./lane.cjs"); - const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); - return { judgeLeg, linesToMessages, isCreditStop }; -})(); +const real = loadLauncher(ROOT); + +test("the files eval-handoff loads are the eval legs' sparse checkout", () => { + for (const name of ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"]) { + const sparse = [...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/sparse-checkout: \|\n((?: {12}\S+\n)+)/g)] + .filter(([block]) => block.includes("run-claude-cli/lane.cjs")) + .map(([, block]) => block.trim().split(/\s+/)); + assert.equal(sparse.length, 3, name); + for (const paths of sparse) assert.deepEqual(paths, CLASSIFIER_FILES, name); + } +}); + +// Run 42138674762: the classifier required in place climbed to the workspace +// root's package.json, which did not parse, and the leg failed before judging. +test("the classifier loads under a workspace whose root package.json does not parse", async () => { + const { spawnSync } = await import("node:child_process"); + const ws = tempDir("eval-handoff-ws-"); + writeFileSync(join(ws, "package.json"), "not json {"); + for (const path of CLASSIFIER_FILES) { + mkdirSync(join(ws, ".ci-workflows", path, ".."), { recursive: true }); + writeFileSync(join(ws, ".ci-workflows", path), readFileSync(join(ROOT, path))); + } + writeFileSync(join(ws, "probe.jsonl"), jsonl(SPEND_LIMIT)); + const r = spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + cwd: ws, + encoding: "utf8", + env: { PATH: process.env.PATH, LEG: "key", CI_WORKFLOWS: join(ws, ".ci-workflows"), PROBE: join(ws, "probe.jsonl"), PROBE_EXIT: "1", RESULTS: "" }, + }); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, "handoff=true\nhandoff-reason=credit-stop\nlane-class=credit-stop\noutcome=failure\n"); +}); + +// A classifier that cannot load fails the run with exit 2 and no verdict, so +// the workflow step stops the leg red rather than guessing a handoff. +test("a classifier that cannot load prints no verdict and exits 2", async () => { + const { spawnSync } = await import("node:child_process"); + const r = spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + encoding: "utf8", + env: { PATH: process.env.PATH, LEG: "key", CI_WORKFLOWS: tempDir("eval-handoff-empty-"), RESULTS: "" }, + }); + assert.equal(r.status, 2); + assert.equal(r.stdout, ""); + assert.match(r.stderr, /^eval-handoff: /); +}); test("every eval leg checks out the classifier at one pin", () => { assert.equal(workflowPins.length, 6, "three legs in each eval workflow"); diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml index 4d97976555f..7de7e6cddda 100644 --- a/.github/workflows/eval-browser-dispatch.yml +++ b/.github/workflows/eval-browser-dispatch.yml @@ -594,7 +594,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -768,7 +769,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -1397,7 +1399,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -1571,7 +1574,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -2154,7 +2158,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -2328,7 +2333,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml index 244615f7698..8d2d7d9f2a8 100644 --- a/.github/workflows/eval-hill-climb-pass.yml +++ b/.github/workflows/eval-hill-climb-pass.yml @@ -347,7 +347,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -411,7 +412,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -738,7 +740,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -802,7 +805,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -1084,7 +1088,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -1148,7 +1153,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi