From 2823bd94f41c67aa432d53cec89af6df209ecfda Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 19:53:49 -0500 Subject: [PATCH] fix(plugin-eval): load the eval handoff classifier under its own package scope In run 42138674762 both measure-skill-edit key legs failed at the probe: eval-handoff.mjs required lane.cjs in place from the .ci-workflows sparse checkout, Node's package-scope lookup climbed past it to the workspace root, and the package.json there did not parse ("Invalid package config"). The classifier never ran, so the legs went red instead of judging or handing off. - loadLauncher copies the three classifier files into a fresh folder whose package.json ends the lookup there, requires them, and removes the folder. - A classifier that cannot run now stops the leg with an explicit ::error in every probe and judge step: fail closed, never a guessed handoff. - Tests: the classifier under a workspace with an unparsable root package.json; a classifier that cannot load prints no verdict and exits 2; the loaded files equal the legs' sparse checkout; a key leg with an empty or credit-skipped key runs no step past its credential check. Refs: #13586 Refs: #12035 Co-Authored-By: Claude Opus 5.5 --- .../actions/resolve-config/callers.test.mjs | 10 ++++ .github/scripts/plugin-eval/eval-handoff.mjs | 33 ++++++++++-- .../scripts/plugin-eval/eval-handoff.test.mjs | 54 +++++++++++++++---- .github/workflows/eval-browser-dispatch.yml | 18 ++++--- .github/workflows/eval-hill-climb-pass.yml | 18 ++++--- 5 files changed, 107 insertions(+), 26 deletions(-) diff --git a/.github/actions/resolve-config/callers.test.mjs b/.github/actions/resolve-config/callers.test.mjs index 830d3a288e1..cd838bee3a4 100644 --- a/.github/actions/resolve-config/callers.test.mjs +++ b/.github/actions/resolve-config/callers.test.mjs @@ -9123,6 +9123,16 @@ for (const [file, gateId, [keyId, subId, fbId], marker] of [ assert.match(step.env.CLAUDE_CODE_SUBPROCESS_ENV_SCRUB, /^(?:"?1"?|\$\{\{ .+ \}\})$/, `${id} ${step.name} scrubs Bash subprocesses`); } assert.equal(steps.find((s) => s.id === "judge").env.LEG, leg, id); + // A leg whose credential is empty, or whose key the gate skipped, runs + // nothing past its credential check: every later step but the budget + // log has a condition, and the first ones read `run`. + const after = steps.slice(steps.findIndex((s) => s.id === "credential") + 1); + assert.deepEqual(after.filter((s) => s.if === undefined || s.if === "always()").map((s) => s.name).filter((n) => n !== "Log the API budget at the end"), [], id); + assert.match(after[0].if, /^steps\.credential\.outputs\.run == 'true'/, id); + // A classifier that cannot run stops the leg red, never as a handoff. + for (const step of steps.filter((s) => /plugin-eval\/eval-handoff\.mjs/.test(String(s.run ?? "")))) { + assert.match(step.run, /eval-handoff\.mjs\) \|\|\n\s+\{ echo "::error::the handoff classifier failed[^"]*"; exit 1; \}/, `${id} ${step.name}`); + } const checkout = steps.find((s) => s.with?.repository === "melodic-software/ci-workflows"); assert.equal(checkout.with.ref, CLASSIFIER_PIN, `${id} reads the classifier at the run-claude-cli pin`); assert.equal(checkout.with["persist-credentials"], false); diff --git a/.github/scripts/plugin-eval/eval-handoff.mjs b/.github/scripts/plugin-eval/eval-handoff.mjs index 4187905dd96..140b205533e 100755 --- a/.github/scripts/plugin-eval/eval-handoff.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.mjs @@ -25,7 +25,7 @@ // one result message carrying the suites' summed durationSeconds and costUsd, // nothing a run wrote). // Prints handoff, handoff-reason, lane-class and outcome as key=value lines. -import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { tmpdir } from "node:os"; import { dirname, isAbsolute, join } from "node:path"; @@ -103,13 +103,36 @@ function readJson(path) { } } +// The classifier's three files, copied under a folder of their own whose +// package.json ends Node's package-scope lookup there. Required in place, the +// lookup climbs past the sparse checkout to the workspace root, and a +// package.json there that does not parse stops the require (run 42138674762). +export const CLASSIFIER_FILES = [ + ".github/actions/run-claude-cli/lane.cjs", + ".github/actions/report-lane-outcome/classify.cjs", + ".github/scripts/resolve-npm-tool-version.cjs", +]; +export function loadLauncher(root) { + const dir = mkdtempSync(join(tmpdir(), "eval-handoff-classifier-")); + try { + writeFileSync(join(dir, "package.json"), '{"type":"commonjs"}\n'); + for (const path of CLASSIFIER_FILES) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + copyFileSync(join(root, path), join(dir, path)); + } + const require = createRequire(join(dir, CLASSIFIER_FILES[0])); + const { judgeLeg, linesToMessages } = require("./lane.cjs"); + const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + return { judgeLeg, linesToMessages, isCreditStop }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + function main(env) { const leg = LEGS[env.LEG]; if (!leg) throw new Error(`LEG must be key, subscription or fallback, not ${JSON.stringify(env.LEG)}`); - const root = env.CI_WORKFLOWS ?? ""; - const require = createRequire(join(root, ".github/actions/run-claude-cli/lane.cjs")); - const { judgeLeg, linesToMessages } = require("./lane.cjs"); - const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); + const { judgeLeg, linesToMessages, isCreditStop } = loadLauncher(env.CI_WORKFLOWS ?? ""); const streams = []; if (env.PROBE && existsSync(env.PROBE)) { const exitCode = Number.parseInt(env.PROBE_EXIT ?? "", 10); diff --git a/.github/scripts/plugin-eval/eval-handoff.test.mjs b/.github/scripts/plugin-eval/eval-handoff.test.mjs index c97127bd997..5c1cc7553a7 100644 --- a/.github/scripts/plugin-eval/eval-handoff.test.mjs +++ b/.github/scripts/plugin-eval/eval-handoff.test.mjs @@ -9,11 +9,10 @@ // records (CLI 2.1.302). import assert from "node:assert/strict"; import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { createRequire } from "node:module"; import { join } from "node:path"; import { test } from "node:test"; import { tempDir } from "../../../lib/test-tmp.mjs"; -import { erroredTraces, handoffReason, judgeLegStreams } from "./eval-handoff.mjs"; +import { CLASSIFIER_FILES, erroredTraces, handoffReason, judgeLegStreams, loadLauncher } from "./eval-handoff.mjs"; test("a key leg hands off on a credit stop or a 401 or 402, never a 403 or a limit stop", () => { assert.equal(handoffReason("api-key", "credit-stop", 400), "credit-stop"); @@ -66,7 +65,6 @@ test("the first stream that hands off decides, else the first stop class is kept // a pin bump that reshapes judgeLeg's return (failureClass, resets, the last // message's api_error_status) or isCreditStop fails here. The three paths are // the legs' sparse checkout. -const CLASSIFIER_FILES = [".github/actions/run-claude-cli/lane.cjs", ".github/actions/report-lane-outcome/classify.cjs", ".github/scripts/resolve-npm-tool-version.cjs"]; const workflowPins = ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"].flatMap((name) => [ ...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/repository: melodic-software\/ci-workflows\n\s+ref: ([0-9a-f]{40})/g), ].map((m) => m[1])); @@ -79,12 +77,50 @@ for (const path of CLASSIFIER_FILES) { mkdirSync(join(ROOT, path, ".."), { recursive: true }); writeFileSync(join(ROOT, path), await response.text()); } -const real = (() => { - const require = createRequire(join(ROOT, ".github/actions/run-claude-cli/lane.cjs")); - const { judgeLeg, linesToMessages } = require("./lane.cjs"); - const { isCreditStop } = require("../report-lane-outcome/classify.cjs"); - return { judgeLeg, linesToMessages, isCreditStop }; -})(); +const real = loadLauncher(ROOT); + +test("the files eval-handoff loads are the eval legs' sparse checkout", () => { + for (const name of ["eval-hill-climb-pass.yml", "eval-browser-dispatch.yml"]) { + const sparse = [...readFileSync(new URL(`../../workflows/${name}`, import.meta.url), "utf8").matchAll(/sparse-checkout: \|\n((?: {12}\S+\n)+)/g)] + .filter(([block]) => block.includes("run-claude-cli/lane.cjs")) + .map(([, block]) => block.trim().split(/\s+/)); + assert.equal(sparse.length, 3, name); + for (const paths of sparse) assert.deepEqual(paths, CLASSIFIER_FILES, name); + } +}); + +// Run 42138674762: the classifier required in place climbed to the workspace +// root's package.json, which did not parse, and the leg failed before judging. +test("the classifier loads under a workspace whose root package.json does not parse", async () => { + const { spawnSync } = await import("node:child_process"); + const ws = tempDir("eval-handoff-ws-"); + writeFileSync(join(ws, "package.json"), "not json {"); + for (const path of CLASSIFIER_FILES) { + mkdirSync(join(ws, ".ci-workflows", path, ".."), { recursive: true }); + writeFileSync(join(ws, ".ci-workflows", path), readFileSync(join(ROOT, path))); + } + writeFileSync(join(ws, "probe.jsonl"), jsonl(SPEND_LIMIT)); + const r = spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + cwd: ws, + encoding: "utf8", + env: { PATH: process.env.PATH, LEG: "key", CI_WORKFLOWS: join(ws, ".ci-workflows"), PROBE: join(ws, "probe.jsonl"), PROBE_EXIT: "1", RESULTS: "" }, + }); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, "handoff=true\nhandoff-reason=credit-stop\nlane-class=credit-stop\noutcome=failure\n"); +}); + +// A classifier that cannot load fails the run with exit 2 and no verdict, so +// the workflow step stops the leg red rather than guessing a handoff. +test("a classifier that cannot load prints no verdict and exits 2", async () => { + const { spawnSync } = await import("node:child_process"); + const r = spawnSync(process.execPath, [new URL("./eval-handoff.mjs", import.meta.url).pathname], { + encoding: "utf8", + env: { PATH: process.env.PATH, LEG: "key", CI_WORKFLOWS: tempDir("eval-handoff-empty-"), RESULTS: "" }, + }); + assert.equal(r.status, 2); + assert.equal(r.stdout, ""); + assert.match(r.stderr, /^eval-handoff: /); +}); test("every eval leg checks out the classifier at one pin", () => { assert.equal(workflowPins.length, 6, "three legs in each eval workflow"); diff --git a/.github/workflows/eval-browser-dispatch.yml b/.github/workflows/eval-browser-dispatch.yml index 4d97976555f..7de7e6cddda 100644 --- a/.github/workflows/eval-browser-dispatch.yml +++ b/.github/workflows/eval-browser-dispatch.yml @@ -594,7 +594,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -768,7 +769,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -1397,7 +1399,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -1571,7 +1574,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -2154,7 +2158,8 @@ jobs: echo "::error::the probe's result holds the credential, so no pull request case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the eval" @@ -2328,7 +2333,8 @@ jobs: [ -n "$dir" ] || continue results+="$OUT/$(basename "$dir")/results.json"$'\n' done <<<"$SUITES" - verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(RESULTS="$results" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi diff --git a/.github/workflows/eval-hill-climb-pass.yml b/.github/workflows/eval-hill-climb-pass.yml index 244615f7698..8d2d7d9f2a8 100644 --- a/.github/workflows/eval-hill-climb-pass.yml +++ b/.github/workflows/eval-hill-climb-pass.yml @@ -347,7 +347,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -411,7 +412,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -738,7 +740,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -802,7 +805,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi @@ -1084,7 +1088,8 @@ jobs: echo "::error::the probe's result holds the credential, so no case runs" exit 1 fi - verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(env -u ANTHROPIC_API_KEY -u CLAUDE_CODE_OAUTH_TOKEN PROBE="$out" PROBE_EXIT="$status" node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } printf '%s\n' "$verdict" >>"$GITHUB_OUTPUT" if grep -qx 'handoff=true' <<<"$verdict"; then echo "::notice::the probe ended on the credential's own stop; the next leg runs the pass" @@ -1148,7 +1153,8 @@ jobs: PROBE_REASON: ${{ steps.probe.outputs.handoff-reason }} PROBE_CLASS: ${{ steps.probe.outputs.lane-class }} run: | - verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) + verdict=$(node .base/.github/scripts/plugin-eval/eval-handoff.mjs) || + { echo "::error::the handoff classifier failed, so this leg cannot tell its credential's stop from a case failure and stops"; exit 1; } if [ "$PROBE_HANDOFF" = true ]; then verdict=$(printf 'handoff=true\nhandoff-reason=%s\nlane-class=%s\noutcome=failure' "$PROBE_REASON" "$PROBE_CLASS") fi