diff --git a/.github/workflows/integration-suite.yml b/.github/workflows/integration-suite.yml index e44db522..0780ea08 100644 --- a/.github/workflows/integration-suite.yml +++ b/.github/workflows/integration-suite.yml @@ -42,6 +42,16 @@ permissions: jobs: integration-suite: runs-on: ubuntu-latest + strategy: + # Two legs. `stable` is the contract test: what users get, and a FAIL fails + # the job. `beta` probes each vendor's public pre-release ref for the CLIs + # that publish one, and is ADVISORY — run.sh exits 0 on it regardless, + # because "this will break on release" is not a reason to stop the world, + # and gating CI on third-party alphas trains everyone to ignore a red run. + # fail-fast: false so a stable failure never cancels the early-warning leg. + fail-fast: false + matrix: + channel: [stable, beta] # NOTE: this is the GitHub *Environment* name where the secrets live. It is # configured in repo settings and is deliberately NOT renamed alongside this # directory — changing it here without renaming it there loses access to @@ -61,12 +71,30 @@ jobs: with: bun-version: latest - - name: Restore integration-suite state (version-gate + broke/recovered) + - name: Restore this leg's state (version-gate + broke/recovered) + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 + with: + path: ${{ matrix.channel == 'stable' && 'integration-suite-state.json' || format('integration-suite-state-{0}.json', matrix.channel) }} + key: integration-suite-state-${{ matrix.channel }}-${{ github.run_id }} + # The channel-scoped prefix first; the unscoped one second so the stable + # leg still adopts the pre-matrix cache on the first run after this + # change, instead of losing its gate baseline and re-probing all 12. + restore-keys: | + integration-suite-state-${{ matrix.channel }}- + integration-suite-state- + + # The beta leg needs the STABLE leg's state to tell "about to break" from + # "already broken" — a CLI that is red on both is the stable leg's alarm, + # not an early warning. Restored read-only; only the stable leg writes it. + - name: Restore stable leg's state (cross-leg comparison) + if: matrix.channel != 'stable' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: integration-suite-state.json - key: integration-suite-state-${{ github.run_id }} - restore-keys: integration-suite-state- + key: integration-suite-state-stable-${{ github.run_id }} + restore-keys: | + integration-suite-state-stable- + integration-suite-state- # TRANSITION (delete once an integration-suite-state-* cache exists — i.e. # after the first successful run on main). This harness was renamed from @@ -76,6 +104,7 @@ jobs: # filename, so it lands as cli-integration-state.json and ci-entrypoint.sh # adopts it only when the new file is absent. - name: Restore legacy state (rename transition) + if: matrix.channel == 'stable' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: cli-integration-state.json @@ -90,6 +119,10 @@ jobs: CANARY_LLM_MODEL: ${{ secrets.CANARY_LLM_MODEL }} CANARY_CLAUDE_MODEL: ${{ secrets.CANARY_CLAUDE_MODEL }} CANARY_PI_MODEL: ${{ secrets.CANARY_PI_MODEL }} + # Optional: unset falls back to ci-entrypoint.sh's default. Present so a + # model that starts refusing codex's payload can be repointed from repo + # settings without a code change (see probe-cli.sh's CANARY_CODEX_MODEL note). + CANARY_CODEX_MODEL: ${{ secrets.CANARY_CODEX_MODEL }} COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} # OAuth credential trees (base64 gzip-tars rooted at $HOME) CURSOR_TOKEN_TGZ_B64: ${{ secrets.CURSOR_TOKEN_TGZ_B64 }} @@ -97,7 +130,13 @@ jobs: ANTIGRAVITY_TOKEN_TGZ_B64: ${{ secrets.ANTIGRAVITY_TOKEN_TGZ_B64 }} # reporting + version-gating CANARY_SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }} - CANARY_STATE: ${{ github.workspace }}/integration-suite-state.json + CANARY_CHANNEL: ${{ matrix.channel }} + CANARY_STATE: ${{ github.workspace }}/${{ matrix.channel == 'stable' && 'integration-suite-state.json' || format('integration-suite-state-{0}.json', matrix.channel) }} + # Operands are FLIPPED deliberately. GHA has no ternary — `cond && a || b` + # is the idiom, but an empty `a` is falsy, so `channel == 'stable' && ''` + # short-circuits to the fallback and the stable leg would get a peer path + # pointing at its own state. Same trap as CANARY_VERSION_GATED above. + CANARY_PEER_STATE: ${{ matrix.channel != 'stable' && format('{0}/integration-suite-state.json', github.workspace) || '' }} CANARY_FP_SHA: ${{ github.sha }} # force=true → "none" (non-"all" ⇒ run.sh gates nothing ⇒ probe all). A # literal '' can't be used: GHA `x && '' || 'all'` is always 'all'. @@ -107,9 +146,12 @@ jobs: CANARY_CLIS: ${{ inputs.clis }} run: bash integration-suite/ci-entrypoint.sh + # Each leg saves ONLY its own file under a channel-scoped key: the legs run + # concurrently, so sharing a key would race, and a beta result must never + # be able to overwrite the stable leg's gating record. - name: Save integration-suite state if: always() uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: - path: integration-suite-state.json - key: integration-suite-state-${{ github.run_id }} + path: ${{ matrix.channel == 'stable' && 'integration-suite-state.json' || format('integration-suite-state-{0}.json', matrix.channel) }} + key: integration-suite-state-${{ matrix.channel }}-${{ github.run_id }} diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b2acc1..fe083fad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.0.15-beta.0 — 2026-07-23 + +### Dependencies +- Bump `next` to 16.2.11 and pin `sharp` to 0.35.0, clearing the ten advisories that turned the Supply Chain gate red on every open PR. Nine are Next.js (four High, CVSS 8.2-8.3: middleware/proxy bypass in App Router, SSRF in Server Actions, and others) fixed in 16.2.11; the tenth is `sharp`'s inherited libvips CVEs (GHSA-f88m-g3jw-g9cj, CVSS 7.0) fixed in 0.35.0. Same shape as the `brace-expansion` incident three days ago: the advisories published on 2026-07-21 and 2026-07-22, *after* `main`'s last green scan at 2026-07-21 14:07 UTC, so every branch went red at once with no dependency change of its own — re-running CI's scanner image against `main`'s unchanged lockfile reproduced the failure identically. The two halves need different tools. `next` is a direct devDependency whose `^16.2.9` range already admitted the fix, so `bun update next` lifts it (and raises the declared floor to `^16.2.11`, so it cannot resolve back). `sharp` cannot be fixed that way: it is an *optional* dependency of `next`, and `next@16.2.11` still declares `sharp: ^0.34.5` — a range that excludes 0.35.0 — so updating `next` leaves the advisory in place and the pin has to come from `overrides`, the same plain-key mechanism already holding `postcss`, `vite`, `undici` and `brace-expansion`. Forcing a *minor* bump of a native library under a dependent asking for `^0.34.5` is the risk here, so it was verified beyond a green lockfile: `sharp` 0.35.0 loads against libvips 8.18.3 and round-trips an encode (`@img/sharp-libvips-linux-x64` moves 1.2.4 → 1.3.0). Confirmed with CI's own scanner image (`ghcr.io/google/osv-scanner-action:v2.3.8`) against the updated lockfile: `No issues found`, exit 0, with `osv-scanner.toml` keeping its zero ignored vulnerabilities. (#591) + +### Features +- Probe each agent CLI's public pre-release build alongside the one users get, so the integration suite reports not only "is enforcement broken?" but "is it about to break, and how long do we have?". The workflow becomes a matrix over `CANARY_CHANNEL`: the `stable` leg is unchanged (all 12 CLIs, a `FAIL` still fails the job) and a new `beta` leg installs each vendor's pre-release ref. The motivating measurement: codex shipped the payload change that broke the 2026-07-22 run in `0.145.0-alpha.4` on 2026-07-11 but did not release it until 2026-07-21 — **10.7 days** of warning, established by installing each alpha and capturing its request body against a mock endpoint. Six CLIs publish something usable: codex `alpha`, copilot `prerelease`, openclaw `beta`, cursor `?channel=lab` (server-validated; `beta`/`canary` are rejected with HTTP 400), goose `CANARY=true` (a *rolling tag* marked `prerelease: false`, so channel detection scanning for `prerelease: true` misses it), and claude — which runs **backwards**: nothing ships ahead of `latest`, so the stable leg now pins `bash -s stable` (~13 days behind, and what conservative users actually run) and `latest` becomes the early-warning ref. That pin also stops a same-day Anthropic release, which lands ~daily, from red-lighting a PR that touched nothing. The other six are deliberately skipped rather than re-installed at stable and reported as coverage: factory, antigravity and pi publish no pre-release at all (verified — antigravity has 19 releases with zero prereleases and its updater serves the identical stable manifest for every `?channel=` value); opencode's `beta`/`dev` tags are ~31 branch snapshots a day rather than release candidates; devin's unpromoted builds exist but cannot be listed, only guessed at; and hermes's installer git-clones `main`, which is also what every hermes user gets, so there is nothing ahead of us to probe. Coverage is stated in the report itself (`watching 6/12`) because a pre-release ref does not preview every release — codex only alphas minor bumps, so all six recent `0.144.x` patches shipped blind, and the copilot 1.0.70 silent-allow had just 0.6 days of lead. (#591) +- Escalate incoming breakage on a cross-leg comparison rather than a beta failure. A CLI red on both legs is already broken and belongs to the stable leg's alarm; the early warning is `stable green + beta not-green`, held for two consecutive runs so an alpha that gets reverted before release doesn't burn attention. This distinction is load-bearing: a vendor payload change stops the model *before* it calls a tool, so it surfaces as `INCONCLUSIVE`/`ERROR` and never as `FAIL` — a FAIL-only rule would have watched the codex regression sail past for ten days in silence. The beta leg is advisory throughout: `run.sh` exits 0 on it regardless, each leg gets its own Docker volume (a pre-release install overwrites the stable binary in a shared `$HOME`) and its own Actions cache key (the legs run concurrently, and a beta result must never overwrite the stable leg's gating record). `__tests__/integration-suite/channel-refs.test.ts` asserts that `run.sh`'s beta CLI list and `install-clis.sh`'s beta refs agree, since a silent drift there would install stable binaries and report them as pre-release coverage — a false all-clear, the worst failure available to an early-warning system. (#591) + +### Fixes +- Stop the workflow handing the stable leg a peer-state path pointing at its own state file, and stop the beta report overstating coverage on a targeted run (both caught in review by CodeRabbit). The first is the GitHub Actions ternary trap the workflow already documents thirty lines above for `CANARY_VERSION_GATED`: Actions has no ternary, `cond && a || b` is the idiom, and an empty `a` is *falsy* — so `matrix.channel == 'stable' && ''` short-circuits to the fallback and both legs received the same path. Latent rather than active (`report.js` only reads the peer state on the beta leg) but a landmine, and fixed by flipping the operands so the true branch is the non-empty one. The second: `run.sh` accepts a CLI subset, so `run.sh cursor` on the beta leg reported "the rest publish no pre-release ref" about CLIs that were merely not requested — precisely the overstated-coverage failure the line exists to prevent. The denominator is now the eligible count passed in from `run.sh`, rather than a hardcoded 12 or a third copy of the CLI list to drift against, so a targeted run reads `watching 2/6 CLIs that publish a pre-release ref (of 12 total)`. Both are covered by new assertions in `__tests__/integration-suite/channel-refs.test.ts`. (#591) +- Repoint the integration suite's codex probe at `gpt-5.1-codex-mini`, restoring the daily enforcement signal for that CLI. The 2026-07-22 run reported `codex bash=INCONCLUSIVE read=INCONCLUSIVE` while the other eleven CLIs stayed green, and the cause was entirely vendor-side: codex-cli shipped 0.145.0 overnight (0.144.6 was green the day before), and for a model it has no metadata for — deepseek logs `Model metadata not found. Defaulting to fallback metadata` — it now sends `reasoning:{summary:"auto"}` and `include:["reasoning.encrypted_content"]` where 0.144.6 sent `reasoning:null` and `include:[]`. The gateway answers `400 "Encrypted content is not supported with this model"` (`param: include`), codex exits before its first tool call, and with no tool call there is no deny to observe, so both probes report INCONCLUSIVE. Enforcement was never broken: the hook log shows `SessionStart` and `UserPromptSubmit` firing in every failed run, and a silent-allow would have surfaced as FAIL, not INCONCLUSIVE. Confirmed by reproducing the CI verdict locally against the real gateway and by replaying the captured request body with and without those two fields — the same body minus `include`/`reasoning` returns 200 on deepseek. It is the same rejection `pi` was already pinned away from over the same `include` param; codex has now grown into it, making three CLIs pinned off the default model. No config override avoids it — `model_reasoning_summary="none"`, `model_supports_reasoning_summaries=false` and `model_reasoning_effort="none"` all still emit `include` — and the escape hatch is gone, since `wire_api = "chat"` is rejected outright by 0.145.0 (openai/codex#7782). `gpt-5.1-codex-mini` is the cheapest gateway model that accepts encrypted reasoning content *and* supports codex's full toolset; `gpt-5.4-nano` accepts the reasoning params but 400s on `tool_search`, which only an end-to-end run reveals. Verified end to end: `bash=PASS read=PASS` with `result=deny policy=custom/canary-bash` and `custom/canary-read` in the oracle. (#591) +- Make `CANARY_CODEX_MODEL` actually reach the probe. `probe-cli.sh` has read the override since the harness was written, but nothing ever set it: only `CANARY_LLM_MODEL`, `CANARY_CLAUDE_MODEL` and `CANARY_PI_MODEL` were written into the container env-file, and the probe runs inside the container, so setting the variable in repo settings would have had no effect at all. `ci-entrypoint.sh` now forwards it and the workflow maps an optional secret, so the next model that starts refusing codex's payload can be swapped from repo settings without a code change. (#591) +- Report a vendor payload rejection as ERROR rather than INCONCLUSIVE. `is_error()` classified quota and auth failures ("can't test right now") apart from a model that simply never called a tool, but a `400` / `invalid_request_error` / `not supported` response fell through to INCONCLUSIVE — so the codex regression above was posted as a quiet 🟡 "all enforcing where the model engaged" for a full day, when the accurate reading was ⚠️ "couldn't test codex". Payload rejections now land in the same bucket as quota errors. The ordering is unchanged and still fail-safe: a deny is checked first, then a leaked side-effect, so a genuine FAIL can never be reclassified. (#591) +- Keep `is_error()`'s payload-rejection patterns machine-shaped, and lock them down with a test. The first cut matched a bare `400` and a bare `not supported`, but the function's input is the agent's *entire transcript* — so "I ran the suite and 400 tests passed" or "that flag is not supported" classified a chatty refusal as a vendor outage, inverting the very signal the change was meant to sharpen (caught in review by CodeRabbit). Now it matches the structured `"code": 400` form and the gateway's own `not supported with` phrasing; both live failures carry `invalid_request_error` and `BadRequestError` regardless, so nothing is lost. `__tests__/integration-suite/is-error.test.ts` runs the real function out of the shell script — not a copy that can drift — against seven vendor-failure fixtures and six ordinary-output fixtures, four of which the previous regex got wrong. (#591) +- Spool probe output to a temp file instead of a shell variable (also from review). Capturing a full agent transcript from up to six CLI invocations into `$(...)` to read one verdict line and a 20-line tail meant a stuck or noisy client could grow the buffer without bound; the previous code streamed through a pipe and never held it. (#591) +- Restore the note explaining why codex is not pinned to a Claude model, which the fix above had deleted along with the line it annotated. The gateway routes Anthropic weighted 1:1 through Bedrock, which 400s on codex's request metadata (#576) — so a Claude-pinned codex probe fails on roughly half its requests, and a coin-flip red in a daily canary is worse than a consistent one. Worth keeping precisely because a single green probe looks like proof that it works. (#591) +- Echo the probe's output tail whenever a CLI's verdict is not a clean pass. `run.sh` discarded everything a probe printed except its `VERDICT_JSON` line, so a yellow or red run said *what* broke and never *why*, and re-running produced no more detail because the vendor's error message was thrown away both times — diagnosing the codex regression above needed a full local reproduction purely for want of these twenty lines. Safe in a public log: every credential involved is a registered Actions secret, so GitHub masks it on the way out. (#591) + ## 0.0.14-beta.3 — 2026-07-20 ### Fixes diff --git a/__tests__/integration-suite/channel-refs.test.ts b/__tests__/integration-suite/channel-refs.test.ts new file mode 100644 index 00000000..3ee26d78 --- /dev/null +++ b/__tests__/integration-suite/channel-refs.test.ts @@ -0,0 +1,98 @@ +/** + * Tripwire for the beta-channel plumbing. + * + * Two lists have to agree or the pre-release leg silently stops meaning anything: + * - install-clis.sh declares a beta install command per CLI (5th arg to `probe`) + * - run.sh hardcodes which CLIs the beta leg probes + * + * If run.sh lists a CLI that install-clis.sh has no beta ref for, the leg probes a + * STABLE binary and reports it as pre-release coverage — a false all-clear, the + * worst possible failure for an early-warning system. If install-clis.sh gains a + * beta ref that run.sh doesn't list, we pay to install it and never probe it. + * + * These are shell files with no importable surface, so the test parses them. That + * is deliberate: the alternative is a third copy of the list to drift against. + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const SUITE = path.join(__dirname, "../../integration-suite"); +const installSh = readFileSync(path.join(SUITE, "install-clis.sh"), "utf8"); +const runSh = readFileSync(path.join(SUITE, "run.sh"), "utf8"); + +/** CLIs whose `probe` line supplies a 5th argument (the beta install command). */ +function cliesWithBetaRef(): string[] { + const out: string[] = []; + // `probe` invocations may span lines via trailing backslashes — join them first. + const joined = installSh.replace(/\\\n\s*/g, " "); + for (const line of joined.split("\n")) { + const m = /^probe\s+(\S+)\s+\S+\s+"[^"]*"\s+(.*)$/.exec(line.trim()); + if (!m) continue; + // Count single-quoted command arguments: 1 = stable only, 2 = stable + beta. + const cmds = m[2].match(/'(?:[^']|'\\'')*'/g) || []; + if (cmds.length >= 2) out.push(m[1]); + } + return out.sort(); +} + +/** The beta-leg CLI list from run.sh. Doubles as the report's coverage denominator. */ +function betaLegClis(): string[] { + const m = /^BETA_CLIS=\(([^)]*)\)/m.exec(runSh); + if (!m) throw new Error("BETA_CLIS not found in run.sh — was it renamed?"); + return m[1].trim().split(/\s+/).sort(); +} + +describe("beta channel refs", () => { + it("finds beta refs in install-clis.sh", () => { + expect(cliesWithBetaRef().length).toBeGreaterThan(0); + }); + + it("run.sh's beta leg probes exactly the CLIs that have a beta ref", () => { + expect(betaLegClis()).toEqual(cliesWithBetaRef()); + }); + + it("covers the CLIs whose pre-release channels were verified live", () => { + // Verified 2026-07-23 against each vendor's registry/CDN. A CLI leaving this + // list means a vendor withdrew a channel — deliberate, but worth a failing + // test so it is a decision rather than an accident. + expect(cliesWithBetaRef()).toEqual(["claude", "codex", "copilot", "cursor", "goose", "openclaw"]); + }); + + it("does not probe CLIs that publish no pre-release ref", () => { + // opencode's beta/dev tags are ~31 branch snapshots a day, not release + // candidates; the rest have nothing at all. Probing any of them on the beta + // leg would install a stable build and report it as pre-release coverage. + for (const cli of ["opencode", "pi", "factory", "devin", "antigravity", "hermes"]) { + expect(cliesWithBetaRef()).not.toContain(cli); + } + }); + + it("pins claude's stable leg to the `stable` channel, not latest", () => { + // claude is the inverted case: `latest` is the bleeding edge and ships ~daily, + // so an unpinned stable leg lets a same-day vendor release red-light an + // unrelated PR. The beta leg is what tracks `latest`. + expect(installSh).toMatch(/probe\s+claude[\s\S]{0,200}?install\.sh \| bash -s stable/); + }); + + it("keeps the beta leg advisory — it must never fail the job", () => { + expect(runSh).toMatch(/if \[ "\$CHANNEL" != stable \]; then[\s\S]{0,300}?exit 0/); + }); + + it("passes the eligible count to report.js so coverage stays honest on targeted runs", () => { + // Without this the beta report says "the rest publish no pre-release ref" + // about CLIs that were merely not requested — overstating coverage, which is + // the one thing an early-warning report must never do. + expect(runSh).toMatch(/report\.js"[^\n]*"\$\{#BETA_CLIS\[@\]\}"/); + }); + + it("does not use a falsy-true-branch ternary for CANARY_PEER_STATE", () => { + // GitHub Actions has no ternary: `cond && '' || fallback` short-circuits to + // the fallback because '' is falsy, so the stable leg would get a peer path + // pointing at its own state. Operands must be flipped instead. + const wf = readFileSync(path.join(__dirname, "../../.github/workflows/integration-suite.yml"), "utf8"); + const line = wf.split("\n").find((l) => l.includes("CANARY_PEER_STATE:")) || ""; + expect(line).not.toMatch(/==\s*'stable'\s*&&\s*''/); + expect(line).toMatch(/!=\s*'stable'\s*&&\s*format\(/); + }); +}); diff --git a/__tests__/integration-suite/is-error.test.ts b/__tests__/integration-suite/is-error.test.ts new file mode 100644 index 00000000..58a124d9 --- /dev/null +++ b/__tests__/integration-suite/is-error.test.ts @@ -0,0 +1,78 @@ +/** + * Tripwire for probe-cli.sh's `is_error()`. + * + * The canary's three-way verdict hangs on this one regex. It decides whether a probe that + * produced no deny is reported as ⚠️ ERROR ("vendor broke, we couldn't test") or 🟡 + * INCONCLUSIVE ("the model just never called a tool") — see report.js's statusOf(). + * + * Its input is the agent's ENTIRE transcript, which is why the negative fixtures matter as + * much as the positive ones: patterns loose enough to match ordinary model prose ("400 + * tests passed", "that flag is not supported") turn a chatty refusal into a fake vendor + * outage, inverting exactly what the function exists to signal. An earlier revision of + * this regex did precisely that with a bare `\b400\b` and a bare `not supported`. + * + * The function is read out of the shell script and executed by bash, so this test tracks + * the real thing rather than a copy that can drift. + */ +import { execFileSync } from "node:child_process"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const PROBE_CLI = path.join(__dirname, "../../integration-suite/probe-cli.sh"); + +/** Pull the one-line `is_error() { ... }` definition straight out of the script. */ +function extractIsError(): string { + const line = readFileSync(PROBE_CLI, "utf8") + .split("\n") + .find((l) => l.startsWith("is_error()")); + if (!line) throw new Error("is_error() not found in probe-cli.sh — was it renamed?"); + return line; +} + +/** Run the real shell function against `output`; true ⇒ classified as a vendor ERROR. */ +function isError(output: string): boolean { + const script = `${extractIsError()}\nif is_error "$1"; then echo yes; else echo no; fi`; + return execFileSync("bash", ["-c", script, "_", output], { encoding: "utf8" }).trim() === "yes"; +} + +describe("probe-cli.sh is_error()", () => { + it("is present and self-contained on one line", () => { + expect(extractIsError()).toMatch(/^is_error\(\) \{.*grep -qiE.*\}$/); + }); + + describe("vendor failures → ERROR", () => { + // Verbatim from the 2026-07-22 codex regression (see CHANGELOG 0.0.15-beta.0). + it.each([ + [ + "codex 0.145.0 encrypted-content 400", + '{"error":{"message":"litellm.BadRequestError: OpenAIException - {\\n \\"error\\": {\\n \\"message\\": \\"Encrypted content is not supported with this model.\\",\\n \\"type\\": \\"invalid_request_error\\",\\n \\"param\\": \\"include\\"\\n}\\n}. Received Model Group=deepseek-v4-pro","code":"400"}}', + ], + [ + "model missing codex's toolset", + "litellm.BadRequestError: OpenAIException - Tool 'tool_search' is not supported with gpt-5.4-nano-2026-03-17.", + ], + ["undeployed model", '{"error":{"code":"DeploymentNotFound","message":"The API deployment ..."}}'], + ["copilot free-tier credits", "You have exceeded your quota. Please upgrade your plan to continue."], + ["expired vendor login", "Error: not logged in. Run `cursor-agent login` first."], + ["throttling", "429 Too Many Requests — please retry later"], + ["bad credential", "401 Unauthorized: invalid api key supplied"], + ])("%s", (_name, output) => { + expect(isError(output)).toBe(true); + }); + }); + + describe("ordinary agent output → NOT an error (stays INCONCLUSIVE)", () => { + // Every one of these was misclassified as ERROR before the patterns were tightened. + it.each([ + ["a number that happens to be 400", "I ran the suite and 400 tests passed."], + ["prose about an unsupported flag", "That command-line flag is not supported by this tool."], + ["explaining an HTTP code", "The endpoint returns 400 when the body is malformed."], + ["a refusal", "I cannot run that; the operation is not supported here."], + ["a plain success", "I ran the command and it completed successfully."], + ["mentioning a rate limiter it read about", "The file describes a limiter, then exits."], + ])("%s", (_name, output) => { + expect(isError(output)).toBe(false); + }); + }); +}); diff --git a/bun.lock b/bun.lock index 8faa374a..e97d38c3 100644 --- a/bun.lock +++ b/bun.lock @@ -28,7 +28,7 @@ "eslint-config-next": "^16.2.9", "jsdom": "^29.1.1", "lucide-react": "^1.18.0", - "next": "^16.2.9", + "next": "^16.2.11", "react": "^19.2.4", "react-dom": "^19.2.4", "tailwind-merge": "^3.4.0", @@ -42,6 +42,7 @@ "brace-expansion": "5.0.7", "eslint-plugin-react-hooks": "7.0.1", "postcss": "8.5.14", + "sharp": "0.35.0", "undici": "7.28.0", "vite": "8.0.16", }, @@ -142,53 +143,57 @@ "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], - "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.3.0" }, "os": "darwin", "cpu": "arm64" }, "sha512-ZgaYEwaj+lx/5n4W8GmZ2IYz0PQHjN5eqRcfijWGB+2Aq7ZInZGa0qJyAn6DEtyLuWHRSrmWOqT9q3qqTBvmUQ=="], - "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.3.0" }, "os": "darwin", "cpu": "x64" }, "sha512-c1z9LFpKB0slQW3RchwBE8iSVzGp70TNjUUO9k4BZwwW4HH7JBGHeIy4b+kk4n/kcBASb9evKCE3/7Slmslgiw=="], - "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + "@img/sharp-freebsd-wasm32": ["@img/sharp-freebsd-wasm32@0.35.0", "", { "dependencies": { "@img/sharp-wasm32": "0.35.0" }, "os": "freebsd" }, "sha512-Li2KTev0H90kEtnJHkI9xQojXt1AqWmFBMXiPw5kqd1jQgP7gi5HVK/qC5Rmh/59NuAwUuPzzPITmX22NomYYQ=="], - "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.3.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-EKbmBKtyTH+GPFDRw2TgK2oV6hyxxlJVIar4hoTYSNmIwipgMFdxPQqR392GmfdsPGWga0mCFN1cCKjRb9cljw=="], - "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.3.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Pl2OmOvrJ42adUllESxBsG54PfXLo1OYg9i3c5/5Ln/qJ0gZuTM9YMhQJPIbXqwidLRc/c2zuHt4RsrymmNv7A=="], - "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.3.0", "", { "os": "linux", "cpu": "arm" }, "sha512-A8UpHoUDW4DwnXoV6+q3C1s7QLRAHtPDEjWuNZjwHMyoCNZnm0GeNN8ls9f/bsEYTRQRW96C/n34XJQHJ2fT7A=="], - "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-C0SqjoFKnszqa44EQ7xoaT48nnO0lOyXEULfXMWi8krrjOPGYkeK30Okzla6ATbBYsyZ0ySinK0FVkpv3DwzfQ=="], - "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.3.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-WOpkVxAjFd369iaIzEgNRreFD+gWdUMIGD5zplhNKNeqS6mm5dac3q2AFyCBmzYoAdouzZvRBgxy4z8QHZb4/A=="], - "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.3.0", "", { "os": "linux", "cpu": "none" }, "sha512-DRWw0mOHusrCCuw2rqP87oLg6PGlkomVDFqw2hIwsSfwWpu4k3XLcBPaKKl6ct/GtL/cwNkgwjV/tc0Mqht3VA=="], - "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.3.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-9APy+nFWhHS+kzLgWZfLcyrUd7YqnAQVa4BPOo4xkoHpdoktOAPG4cEr9+Jpl0TtqfVmcMJimNL5qNTyyOHZNA=="], - "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-y9RNUYDe2A1UAdhLyfeOodGRszQdaEoe4nfOpp/sNVPl2CWIcUyFaDoCh4vPLPxu19803j2naLqZup2WxDXCLA=="], - "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.3.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-cC1wkC0Mlucd0KSiGrLkJnB/ZqPvZCntc/Lk7ZnYO5ZSbF2euNek4Xvxafojq+wN1q/W0eprdpUIjUr/EV2PBg=="], - "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.3.0", "", { "os": "linux", "cpu": "x64" }, "sha512-LiYMhUZicB1QG//+RvmYZpXJO8fYRENfp+MZUCnG9aw+AKvGAy9gPaCnuwsPcBFs8EV66M0NNxj9VHcNklE8zw=="], - "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.3.0" }, "os": "linux", "cpu": "arm" }, "sha512-VVlpEWwizEFIOom0zdoeKuO5nuTswzVE5uHcBNvHzmeHUpNFajY3HFfbQ+zIH4E2kVaZ/yVxmsShW56TtEy4uA=="], - "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.3.0" }, "os": "linux", "cpu": "arm64" }, "sha512-4+4XHLNT5wDT0roYlHTEmH9lDKt0acf9Tv+3hM3iceOirkxrR404/3WjAYZ9F9CkHrxeRcGLJXbi4vluMZ9O+A=="], - "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.3.0" }, "os": "linux", "cpu": "ppc64" }, "sha512-N3hzbEpUTJC8pWpPVJvgzGxM+so/MAXc8O2s/53B0LL9ZGpfXpME7Wizkc5d/8fRBlBtkDjzoZGDCqqNDHqLEw=="], - "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.3.0" }, "os": "linux", "cpu": "none" }, "sha512-l6vmKVPnbS0RhVMbyxP5meAARsbhCnBN4fy31qz0+3a6Rv4jEqfzDrT89y6ZPkCi0AJGnwp2En528yXo401Hpw=="], - "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.3.0" }, "os": "linux", "cpu": "s390x" }, "sha512-MYlMiPFiv/EKPAHnp3yNZ9AAWFsxga9c5Bkc6wkar6bqzHLlkGVJHRm0u1ei+VXnZxp3Mz9MG9ZIsI8vSOf3sQ=="], - "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.3.0" }, "os": "linux", "cpu": "x64" }, "sha512-TYaItB5oj1ioXjhyn2xrR208vf+YuIIcHptQWRRaBmFhvIvL9D72DXN8w75xup0KXA8UdEAhQ9Qb2S49FD/9Cw=="], - "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.3.0" }, "os": "linux", "cpu": "arm64" }, "sha512-DSTb6ijQzqe6DdAaOBVqJ/SYf1vO8EW5bK6X6LRXufEBebf2722VCdvBUtZ3rtV0x2ApfPNDy/p7LrrjaWjiyQ=="], - "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.35.0", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.3.0" }, "os": "linux", "cpu": "x64" }, "sha512-K7ykQ+26Rt6+4BTU80AuGgTPIYX86UxiAKT4rcXX/WNTo7k1ZxpKz+TguHnwVpCqQK3B5PK0vZ0ZBe6nz/ib1w=="], - "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.35.0", "", { "dependencies": { "@emnapi/runtime": "^1.11.0" } }, "sha512-9woLIFORERCr+6cWu87dQ22J34EExkhc73U1kZW0c+RclQqWetoodByp4dWZ/hN8/KVmTRAx2HOnUwib8AwZdA=="], - "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + "@img/sharp-webcontainers-wasm32": ["@img/sharp-webcontainers-wasm32@0.35.0", "", { "dependencies": { "@img/sharp-wasm32": "0.35.0" }, "cpu": "none" }, "sha512-t+kie1TOyaDM6Dho+f+y0VqIUNhYQaKCUahuZVi0E0frgdiaOaPsDxDW3wfKacUdaNBCnK/ZDBMg33ydvHj8uA=="], - "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.35.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-M5eKxug0dabbaWgFKvPa3odNs2OpaP+81NASfGKkt4GcYXpNhSu7CaeYxWkLNV6vHmUp4hnCxnxrUyhUJhXbKA=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.35.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-z0+pZ03QCDvdVN0Ez9IX/yjWC19ikMlXrmdYMwYNLTh2BLPx3hXWPvyqWfquZ0BTO9O6GVOjIVoTcyyacMnWlQ=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.35.0", "", { "os": "win32", "cpu": "x64" }, "sha512-feNnlz5ZHKr0MY1LPHvZQyJeBkbo4ctsn0D8FvA53VTw5TC63rfEL2UrWbkSBR19htSE7Mw78xYVwdJqoMWVHw=="], "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.13", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA=="], @@ -204,25 +209,25 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@1.1.4", "", { "dependencies": { "@tybys/wasm-util": "^0.10.1" }, "peerDependencies": { "@emnapi/core": "^1.7.1", "@emnapi/runtime": "^1.7.1" } }, "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow=="], - "@next/env": ["@next/env@16.2.10", "", {}, "sha512-zLPxg9M0MEHmygpj5OuxjQ+vHMiy/K7cSp74G8ecYolmgUWw0RwN02tF56npup/+qaI8JB97hQgS/r2Hb6QwVA=="], + "@next/env": ["@next/env@16.2.11", "", {}, "sha512-0do5A3BJ2gxWr0ZCMcD6BhW+e595jyxdTl3rXTS6lOtD8ektMiW6CO+EPwt1Eca1DBnm90r/7GdiKWBKxH++DA=="], "@next/eslint-plugin-next": ["@next/eslint-plugin-next@16.2.10", "", { "dependencies": { "fast-glob": "3.3.1" } }, "sha512-Gs8D2m21VnJeFo9qvYIIqJH94frWerWYu41BprU1pLtRVF7PCQNLiFZZ3fG+iPuj3K83Cwv/rt+msLOy8Qgu3Q=="], - "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.10", "", { "os": "darwin", "cpu": "arm64" }, "sha512-v9IdJCa0H0mbo+8z5zwUpOk1Vj7RjkcI5uNYf5Ws1y6szf/p3Mzl9hLaST8SCt6L9h8NGnruZcd2+o0NTNwDhA=="], + "@next/swc-darwin-arm64": ["@next/swc-darwin-arm64@16.2.11", "", { "os": "darwin", "cpu": "arm64" }, "sha512-wryL4pjKmDwGv2ox6+GZDFxvmtSRLqApBR8kL1j4+vhB7Z5vJC/zAnXpiR9Xkfzl0AS8WLMnsuGV/UKI67/rrw=="], - "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.10", "", { "os": "darwin", "cpu": "x64" }, "sha512-17IS0jJRViROGmA9uGdNR8VPJpfbnaVG7E9qhso5jDLkmyd0lSDORWxbcKINzcFqzZqGwGtMSnrFRxBpuUYjLQ=="], + "@next/swc-darwin-x64": ["@next/swc-darwin-x64@16.2.11", "", { "os": "darwin", "cpu": "x64" }, "sha512-aZl2j4f/fLyjQvOhv0Oe9UaMAQHolYpKhctsoYzplSumKJKPUmgjcf6545aBtysLTcu994TREd0+pSgNE4ohmg=="], - "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-GRQRsRtuciNJvB54AvvuQTiq0oZtFwa1owQqtZD8wwnGpM2L39MV22kpI72YSXLKIyY40LC66EiLFv4PiicXxg=="], + "@next/swc-linux-arm64-gnu": ["@next/swc-linux-arm64-gnu@16.2.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-5jEriyEnH/LWFy27L2ZG0XaLlyEJIjhsImEsiS9P563PKEVp2BVups/xfOucIrsvVntp11oNcZwjHvaDPYVB5g=="], - "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.10", "", { "os": "linux", "cpu": "arm64" }, "sha512-zkN9MQYS7UQBro+FnISUq1itaQjXI9xqISzuQ+2bc921NcJ1x4yPCqrn77tVN6/dOOXaaWVX3k6/bR07pPwK+A=="], + "@next/swc-linux-arm64-musl": ["@next/swc-linux-arm64-musl@16.2.11", "", { "os": "linux", "cpu": "arm64" }, "sha512-eIjcpx2fnnFSSkZDbTxy74KnokUXDjfoLClpWelfgHLf621aTqswhwXQ7GkD5K5rplrS6LZ/Bj+mVuvzluBOEg=="], - "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-iCVJnwvrPYECvA6WM/7+oo+OiTvedIKLxtCLAZP4xZR3nXa1zmzZyLPbYCmWvpd4CvMYF1EMTafd0ii3DygLvA=="], + "@next/swc-linux-x64-gnu": ["@next/swc-linux-x64-gnu@16.2.11", "", { "os": "linux", "cpu": "x64" }, "sha512-8WgzpaWMs46qJT9kiV47cje86L0x/Mu9t8/Gwj+pnbgW3rETVfCnaScPjlYUwNScpOozdcIMHWmAvuZJUonR2w=="], - "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.10", "", { "os": "linux", "cpu": "x64" }, "sha512-ov2g4H0dHY9bPoOU83m91hWT7Iq5qy13bUnyyshLU3HGR1Ownn0X9QpmDPc5iIUaahTp7f7LeGAhV4DSFtackw=="], + "@next/swc-linux-x64-musl": ["@next/swc-linux-x64-musl@16.2.11", "", { "os": "linux", "cpu": "x64" }, "sha512-I3UgPds7G4ZYnTb/H+5GBGuUT2DhAk6j0mL6A4s63RjFs74wB2hOWP0vaxsK+3NJraExt3eYEPQ/UtT0x/64Nw=="], - "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.10", "", { "os": "win32", "cpu": "arm64" }, "sha512-DwAnhLX76HQiFFQNgWlcK+JzlnD1rZ+UK/WY0ZMI/deXpvgnesjNYrqcfo1JzBuz4Kf7o3brIBL0glI1junatA=="], + "@next/swc-win32-arm64-msvc": ["@next/swc-win32-arm64-msvc@16.2.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-n89CjtcThnjrwgJMAiI5xbqwLY51zvwC9tSlArmVndAJLYVl9T9UAdlkXTmZvE++idoXe8KdglQlhNRdUp1c6g=="], - "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.10", "", { "os": "win32", "cpu": "x64" }, "sha512-0JXq3b85Jk9Jg4ntLUbXSPvoDw3gpZou7twuKdoFG2jOw635v7+IiXfTaa0TxVMyx78pUjnrVYwLgjKfX4e6/A=="], + "@next/swc-win32-x64-msvc": ["@next/swc-win32-x64-msvc@16.2.11", "", { "os": "win32", "cpu": "x64" }, "sha512-md8CLNggS1Dx9pUgApzps5uAf+N8GN9xywzmNx9vHAWo94HtBwCCqkSnhIrdfQe83Dhz8Lfo/20Nb1Zxal092w=="], "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], @@ -982,7 +987,7 @@ "natural-compare": ["natural-compare@1.4.0", "", {}, "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw=="], - "next": ["next@16.2.10", "", { "dependencies": { "@next/env": "16.2.10", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.10", "@next/swc-darwin-x64": "16.2.10", "@next/swc-linux-arm64-gnu": "16.2.10", "@next/swc-linux-arm64-musl": "16.2.10", "@next/swc-linux-x64-gnu": "16.2.10", "@next/swc-linux-x64-musl": "16.2.10", "@next/swc-win32-arm64-msvc": "16.2.10", "@next/swc-win32-x64-msvc": "16.2.10", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-2som5AVXb3kE6Yjine3/mNbBayYF58eguBWIVVUdr1y/L426xyVEgYxgBG+1QC34P2x5E+tcDup6XkuOAX3dCA=="], + "next": ["next@16.2.11", "", { "dependencies": { "@next/env": "16.2.11", "@swc/helpers": "0.5.15", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", "postcss": "8.4.31", "styled-jsx": "5.1.6" }, "optionalDependencies": { "@next/swc-darwin-arm64": "16.2.11", "@next/swc-darwin-x64": "16.2.11", "@next/swc-linux-arm64-gnu": "16.2.11", "@next/swc-linux-arm64-musl": "16.2.11", "@next/swc-linux-x64-gnu": "16.2.11", "@next/swc-linux-x64-musl": "16.2.11", "@next/swc-win32-arm64-msvc": "16.2.11", "@next/swc-win32-x64-msvc": "16.2.11", "sharp": "^0.34.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", "@playwright/test": "^1.51.1", "babel-plugin-react-compiler": "*", "react": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "react-dom": "^18.2.0 || 19.0.0-rc-de68d2f4-20241204 || ^19.0.0", "sass": "^1.3.0" }, "optionalPeers": ["@opentelemetry/api", "@playwright/test", "babel-plugin-react-compiler", "sass"], "bin": { "next": "dist/bin/next" } }, "sha512-B339zaqbyK8cmxhoAvLrcwoabwCP1wz21zSzfqxqXAemTu2BXnH7tQnfcglKv1vnMUIDBc+Hth7XODQriTZiRQ=="], "node-exports-info": ["node-exports-info@1.6.0", "", { "dependencies": { "array.prototype.flatmap": "^1.3.3", "es-errors": "^1.3.0", "object.entries": "^1.1.9", "semver": "^6.3.1" } }, "sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw=="], @@ -1106,7 +1111,7 @@ "set-proto": ["set-proto@1.0.0", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.0.0" } }, "sha512-RJRdvCo6IAnPdsvP/7m6bsQqNnn1FCBX5ZNtFL98MmFF/4xAIJTIg1YbHW5DC2W5SKZanrC6i4HsJqlajw/dZw=="], - "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "sharp": ["sharp@0.35.0", "", { "dependencies": { "@img/colour": "^1.1.0", "detect-libc": "^2.1.2", "semver": "^7.8.4" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.35.0", "@img/sharp-darwin-x64": "0.35.0", "@img/sharp-freebsd-wasm32": "0.35.0", "@img/sharp-libvips-darwin-arm64": "1.3.0", "@img/sharp-libvips-darwin-x64": "1.3.0", "@img/sharp-libvips-linux-arm": "1.3.0", "@img/sharp-libvips-linux-arm64": "1.3.0", "@img/sharp-libvips-linux-ppc64": "1.3.0", "@img/sharp-libvips-linux-riscv64": "1.3.0", "@img/sharp-libvips-linux-s390x": "1.3.0", "@img/sharp-libvips-linux-x64": "1.3.0", "@img/sharp-libvips-linuxmusl-arm64": "1.3.0", "@img/sharp-libvips-linuxmusl-x64": "1.3.0", "@img/sharp-linux-arm": "0.35.0", "@img/sharp-linux-arm64": "0.35.0", "@img/sharp-linux-ppc64": "0.35.0", "@img/sharp-linux-riscv64": "0.35.0", "@img/sharp-linux-s390x": "0.35.0", "@img/sharp-linux-x64": "0.35.0", "@img/sharp-linuxmusl-arm64": "0.35.0", "@img/sharp-linuxmusl-x64": "0.35.0", "@img/sharp-webcontainers-wasm32": "0.35.0", "@img/sharp-win32-arm64": "0.35.0", "@img/sharp-win32-ia32": "0.35.0", "@img/sharp-win32-x64": "0.35.0" } }, "sha512-BqvG5XbwPZ4NV0DK90d86leEECMsoa8bO0nqnKWlBDYxri4GJ7c4EDInaF6q20lTh/mATmnDIKWJFfXnoVfH5g=="], "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], @@ -1300,6 +1305,8 @@ "@eslint-community/eslint-utils/eslint-visitor-keys": ["eslint-visitor-keys@3.4.3", "", {}, "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag=="], + "@img/sharp-wasm32/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.11.2", "", { "dependencies": { "@emnapi/wasi-threads": "1.2.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-TC8MkTuZUtcTSiFeuC0ksCh9QIJ5+F21MvZ4Wn4ORfYaFJ/0dsiudv5tVkejgwZlwQ39jL9WWDe2lz8x0WglOA=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.11.2", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-kyOl3X0DuTiT1h2ft8r2fYO8JYtU9a9Xis/zBSiGArNaagCOWx90N1k2wxp18czFDH+OgcWGb5ZP/XMt3dcyPA=="], @@ -1346,6 +1353,6 @@ "prop-types/react-is": ["react-is@16.13.1", "", {}, "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ=="], - "sharp/semver": ["semver@7.8.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-rkVq3IXh+4FDGch+KwzX3aV9W3kO54GyEgpvBzSyctDA6Xtd7RJQV1xmXbeQp5v7+VzLOfVqiutSE6GICgPFvg=="], + "sharp/semver": ["semver@7.8.5", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA=="], } } diff --git a/integration-suite/README.md b/integration-suite/README.md index 0840f100..b98ea94d 100644 --- a/integration-suite/README.md +++ b/integration-suite/README.md @@ -45,6 +45,48 @@ failproofai's HEAD changed since its last green run — unchanged ⇒ can't have drifted ⇒ skip, protecting LLM/vendor quota. `FAIL`/`INCONCLUSIVE`/`ERROR` always re-probe until they recover. Dispatch with **force** to probe everything. +## Two channels: `stable` and `beta` + +The workflow runs as a **matrix over `CANARY_CHANNEL`**, in two concurrent legs +answering different questions: + +| Leg | Installs | Question | On failure | +|-----|----------|----------|------------| +| `stable` | what users get, all 12 CLIs | *is enforcement broken now?* | **fails the job** + Slack | +| `beta` | each vendor's public pre-release ref, 6 CLIs | *is it about to break?* | **advisory only** — never fails | + +Only six vendors publish something usable ahead of their release, so the beta leg +skips the rest rather than re-installing a stable build and reporting it as +pre-release coverage: + +| CLI | Pre-release ref | Typical lead | +|-----|-----------------|--------------| +| codex | npm `alpha` | up to ~12 d (minor bumps only — patches ship blind) | +| copilot | npm `prerelease` | 0.6–5.8 d | +| openclaw | npm `beta` | 0.3–11.4 d | +| cursor | `install?channel=lab` | 2–4 d | +| goose | `CANARY=true` (rolling tag) | ~1 release cycle | +| claude | `latest` — **inverted**, see below | ~13 d | + +**claude runs backwards.** Nothing ships ahead of `latest`, which *is* the +bleeding edge (~1 release/day); what exists is `stable`, ~13 days behind. So the +stable leg pins `bash -s stable` — testing what conservative users run, and +stopping a same-day Anthropic release from red-lighting an unrelated PR — and +`latest` becomes the early-warning ref. **hermes** is inverted too but has no fix: +its installer git-clones `main`, which is also what every hermes user gets, so +there is nothing ahead of us to probe. + +**Escalation is a cross-leg comparison, not a beta failure.** A CLI red on both +legs is already broken and belongs to the stable leg's alarm. The early warning is +`stable green + beta not-green`, held for **two consecutive runs** so a broken +alpha that gets reverted before release doesn't burn attention. Note the signal is +usually `INCONCLUSIVE`/`ERROR`, not `FAIL` — a vendor payload change stops the +model before it ever calls a tool — so a FAIL-only rule would miss it entirely. + +Each leg has its own Docker volume and Actions cache key: a pre-release install +overwrites the stable binary in a shared `$HOME`, and a beta result must never be +able to overwrite the stable leg's gating record. + ## Auth & secrets (the `cli-integration` Environment) > The GitHub **Environment** is still named `cli-integration` — it's configured in @@ -77,7 +119,7 @@ re-login on the capture machine and refresh the secret. ``` ci-entrypoint.sh CI front door: build -> sandbox -> install -> probe -> cleanup Dockerfile non-root sandbox base image -install-clis.sh install/upgrade all 12 CLIs @latest (with retries) +install-clis.sh install/upgrade the CLIs for $CANARY_CHANNEL (with retries) inject-tokens.sh write captured OAuth creds into the fresh volume probe-cli.sh live enforcement probe for ONE CLI (the oracle) canary-policies.mjs benign-marker custom policies the probe trips diff --git a/integration-suite/ci-entrypoint.sh b/integration-suite/ci-entrypoint.sh index 12ca4ba6..0a62f616 100755 --- a/integration-suite/ci-entrypoint.sh +++ b/integration-suite/ci-entrypoint.sh @@ -27,6 +27,7 @@ # CANARY_LLM_MODEL (default deepseek-v4-pro) # CANARY_CLAUDE_MODEL (default claude-haiku-4-5) # CANARY_PI_MODEL (default claude-haiku-4-5) +# CANARY_CODEX_MODEL (default gpt-5.1-codex-mini) # COPILOT_GITHUB_TOKEN copilot PAT # CURSOR_TOKEN_TGZ_B64 } base64 gzip-tars of each OAuth credential tree, # DEVIN_TOKEN_TGZ_B64 } rooted at $HOME (see capture-tokens.sh) @@ -43,11 +44,20 @@ set -u HERE="$(cd "$(dirname "$0")" && pwd)" REPO="${GITHUB_WORKSPACE:-$(dirname "$HERE")}" -VOL="${CANARY_VOL:-integration-suite}" +# Each channel gets its OWN volume: installing a pre-release into the same $HOME +# overwrites the stable binary (npm -g, and the vendor installers all rewrite +# their own shims), so the two refs cannot coexist in one volume. +CHANNEL="${CANARY_CHANNEL:-stable}" +SUFFIX=""; [ "$CHANNEL" != stable ] && SUFFIX="-$CHANNEL" +VOL="${CANARY_VOL:-integration-suite$SUFFIX}" IMAGE="${CANARY_IMAGE:-failproofai-integration-suite:base}" -ENVFILE="${CANARY_ENVFILE:-$REPO/canary.env}" -TOKENS_DIR="${CANARY_TOKENS_DIR:-$REPO/tokens}" -STATE="${CANARY_STATE:-$REPO/integration-suite-state.json}" +ENVFILE="${CANARY_ENVFILE:-$REPO/canary$SUFFIX.env}" +TOKENS_DIR="${CANARY_TOKENS_DIR:-$REPO/tokens$SUFFIX}" +STATE="${CANARY_STATE:-$REPO/integration-suite-state$SUFFIX.json}" +# The stable leg's state, so the beta leg can tell "about to break" from +# "already broken" (see report.js). Unset on the stable leg. +PEER_STATE="${CANARY_PEER_STATE:-}" +[ "$CHANNEL" != stable ] && [ -z "$PEER_STATE" ] && PEER_STATE="$REPO/integration-suite-state.json" step() { echo "── $* ──" >&2; } @@ -123,8 +133,8 @@ docker volume rm "$VOL" -f >/dev/null 2>&1 || true docker volume create "$VOL" >/dev/null || { echo "✗ volume create failed" >&2; exit 1; } # ── 4. install CLIs @latest, then inject tokens ───────────────────────────── -step "installing 12 CLIs @latest into the fresh volume" -docker run --rm -v "$VOL:/home/canary" -v "$HERE:/opt/canary:ro" \ +step "installing CLIs (channel=$CHANNEL) into the fresh volume" +docker run --rm -e CANARY_CHANNEL="$CHANNEL" -v "$VOL:/home/canary" -v "$HERE:/opt/canary:ro" \ "$IMAGE" bash /opt/canary/install-clis.sh \ || { echo "✗ CLI install failed" >&2; exit 1; } @@ -145,6 +155,9 @@ step "assembling gateway env-file" echo "CANARY_LLM_MODEL=${CANARY_LLM_MODEL:-deepseek-v4-pro}" echo "CANARY_CLAUDE_MODEL=${CANARY_CLAUDE_MODEL:-claude-haiku-4-5}" echo "CANARY_PI_MODEL=${CANARY_PI_MODEL:-claude-haiku-4-5}" + # Without this line probe-cli.sh's CANARY_CODEX_MODEL override was dead plumbing: + # the var is read inside the container, and only what is written here crosses into it. + echo "CANARY_CODEX_MODEL=${CANARY_CODEX_MODEL:-gpt-5.1-codex-mini}" echo "COPILOT_GITHUB_TOKEN=${COPILOT_GITHUB_TOKEN:-}" } > "$ENVFILE" ) @@ -161,4 +174,6 @@ CANARY_VOL="$VOL" \ CANARY_IMAGE="$IMAGE" \ CANARY_STATE="$STATE" \ CANARY_ENVFILE="$ENVFILE" \ +CANARY_CHANNEL="$CHANNEL" \ +CANARY_PEER_STATE="$PEER_STATE" \ bash "$HERE/run.sh" ${CANARY_CLIS:-} diff --git a/integration-suite/install-clis.sh b/integration-suite/install-clis.sh index 1d3f8213..aa812a7e 100644 --- a/integration-suite/install-clis.sh +++ b/integration-suite/install-clis.sh @@ -2,10 +2,22 @@ # ───────────────────────────────────────────────────────────────────────────── # Tier-0 install probe — runs INSIDE the canary sandbox container. # -# Installs/upgrades all 12 agent CLIs to @latest into the persistent HOME volume, -# then records which resolve to a runnable binary + their version. This is both -# the daily "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary -# breaks show up here as a FAIL). +# Installs/upgrades the agent CLIs into the persistent HOME volume, then records +# which resolve to a runnable binary + their version. This is both the daily +# "fresh upgrade" step AND the Tier-0 canary signal (packaging/binary breaks show +# up here as a FAIL). +# +# CANARY_CHANNEL selects WHICH ref of each CLI to install: +# stable (default) — what users get. All 12 CLIs. +# beta — the vendor's public pre-release ref, for early warning. +# Only the CLIs that actually publish one; the rest are +# SKIPPED, never silently re-installed at stable (that would +# be duplicate work reported as coverage). +# +# Every beta ref below was verified live against the vendor's registry/CDN on +# 2026-07-23; see ~/Desktop/failproofai-integration-suite-beta-channels-design.md +# for the evidence, lead times, and the CLIs that have no pre-release ref at all +# (factory, antigravity, pi, opencode, devin, hermes). # # Install methods verified 2026-07-16 (see failproofai-cli-canary-design.md). # Vendor installers scatter binaries across several dirs, so we search an @@ -18,6 +30,7 @@ set -u RESULTS="${CANARY_RESULTS:-$HOME/canary-tier0.json}" LOGDIR="${CANARY_LOGDIR:-$HOME/canary-logs}" +CHANNEL="${CANARY_CHANNEL:-stable}" mkdir -p "$LOGDIR" # Cover every dir the vendor installers are known to drop binaries into. @@ -25,8 +38,19 @@ export PATH="$HOME/.npm-global/bin:$HOME/.local/bin:$HOME/.factory/bin:$HOME/.he ROWS=() +# probe <--version flag> [beta install cmd] +# Omitting the 5th argument declares "this CLI has no public pre-release ref", +# which skips it entirely on the beta channel. probe() { - local id="$1" bin="$2" vflag="$3" install="$4" + local id="$1" bin="$2" vflag="$3" install="$4" beta="${5:-}" + if [ "$CHANNEL" != stable ]; then + if [ -z "$beta" ]; then + echo "════════════════════ $id ════════════════════" + echo " skipped — no public pre-release ref for this CLI" + return 0 + fi + install="$beta" + fi echo "════════════════════ $id ════════════════════" local t0 t1 rc=1; t0=$SECONDS # Retry transient install failures. Flaky vendor CDNs / HTTP-2 stream resets are @@ -64,19 +88,40 @@ probe() { "$id" "$status" "${resolved:-}" "${ver:-}" "$rc" "$((t1 - t0))")") } -# id binary --ver flag install command (@latest) -probe claude claude "--version" 'curl -fsSL https://claude.ai/install.sh | bash' -probe codex codex "--version" 'npm install -g @openai/codex@latest' -probe copilot copilot "--version" 'npm install -g @github/copilot@latest' -probe cursor cursor-agent "--version" 'curl https://cursor.com/install -fsS | bash' +# id binary --ver flag stable ref beta ref (omitted = none exists) +# +# claude is the one CLI whose channels run BACKWARDS: nothing ships ahead of +# `latest`, which is the bleeding edge (~1 release/day). What exists is `stable`, +# ~13 days behind. So the stable leg pins `stable` — both to test what +# conservative users actually run and to stop a same-day Anthropic release +# red-lighting an unrelated PR — and `latest` becomes the early-warning ref. +probe claude claude "--version" 'curl -fsSL https://claude.ai/install.sh | bash -s stable' \ + 'curl -fsSL https://claude.ai/install.sh | bash' +probe codex codex "--version" 'npm install -g @openai/codex@latest' \ + 'npm install -g @openai/codex@alpha' +probe copilot copilot "--version" 'npm install -g @github/copilot@latest' \ + 'npm install -g @github/copilot@prerelease' +# Cursor validates the channel server-side: only prod/lab/static/prod-stable-internal +# return 200, anything else is a 400. `lab` is Anysphere's own dogfood channel. +probe cursor cursor-agent "--version" 'curl https://cursor.com/install -fsS | bash' \ + 'curl "https://cursor.com/install?channel=lab" -fsS | bash' +# opencode's beta/dev tags are ~31 branch snapshots a day (0.0.0--), +# not release candidates — deliberately no beta ref. probe opencode opencode "--version" 'npm install -g opencode-ai@latest' probe pi pi "--version" 'npm install -g @mariozechner/pi-coding-agent@latest' +# hermes install.sh git-clones main, which is ALSO what every hermes user gets — +# its tags are markers nobody installs. So there is nothing ahead of us to probe, +# and pinning a tag would test something no user runs. probe hermes hermes "--version" 'curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash' -probe openclaw openclaw "--version" 'npm install -g openclaw@latest' +probe openclaw openclaw "--version" 'npm install -g openclaw@latest' \ + 'npm install -g openclaw@beta' probe factory droid "--version" 'curl -fsSL https://app.factory.ai/cli | sh' probe devin devin "--version" 'curl -fsSL https://cli.devin.ai/install.sh | bash' probe antigravity agy "--version" 'curl -fsSL https://antigravity.google/cli/install.sh | bash' -probe goose goose "--version" 'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash' +# goose's canary is a ROLLING TAG (prerelease:false), selected by env var — not a +# GitHub prerelease, so channel detection that scans for prerelease:true misses it. +probe goose goose "--version" 'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CONFIGURE=false bash' \ + 'curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | CANARY=true CONFIGURE=false bash' # Emit JSON array. { printf '['; IFS=,; printf '%s' "${ROWS[*]}"; printf ']\n'; } > "$RESULTS" @@ -87,5 +132,5 @@ ok=0; fail=0 for r in "${ROWS[@]}"; do case "$r" in *'"status":"OK"'*) ok=$((ok+1));; *) fail=$((fail+1));; esac done -printf 'installed OK: %s / 12 failed: %s\n' "$ok" "$fail" +printf 'channel: %s installed OK: %s / %s failed: %s\n' "$CHANNEL" "$ok" "$((ok + fail))" "$fail" echo "results JSON: $RESULTS" diff --git a/integration-suite/probe-cli.sh b/integration-suite/probe-cli.sh index 556efefe..adcb70ad 100644 --- a/integration-suite/probe-cli.sh +++ b/integration-suite/probe-cli.sh @@ -14,14 +14,33 @@ set -u CLI="${1:?usage: probe-cli.sh }" : "${CANARY_LLM_API_KEY:?gateway key missing}" -# Gateway default model. deepseek-v4-pro is cheapest AND works on both the OpenAI -# chat-completions and Responses paths. EXCEPTION: the claude CLI speaks Anthropic -# tool-use, which deepseek-via-/v1/messages doesn't emit correctly (all probes go -# INCONCLUSIVE), so claude pins to the cheapest Anthropic model via CANARY_CLAUDE_MODEL. +# Gateway default model, used by every CLI without a pin of its own. deepseek-v4-pro is +# cheapest AND works on both the OpenAI chat-completions and Responses paths. Three CLIs +# are pinned away from it below (claude, pi, codex) — each for a payload deepseek refuses. +# EXCEPTION 1: the claude CLI speaks Anthropic tool-use, which deepseek-via-/v1/messages +# doesn't emit correctly (all probes go INCONCLUSIVE), so claude pins to the cheapest +# Anthropic model via CANARY_CLAUDE_MODEL. : "${CANARY_LLM_MODEL:=deepseek-v4-pro}" : "${CANARY_CLAUDE_MODEL:=claude-haiku-4-5}" -# pi sends an `include` (encrypted reasoning) param deepseek rejects (400) → pin to Anthropic. +# EXCEPTION 2: pi sends an `include` (encrypted reasoning) param deepseek rejects (400). : "${CANARY_PI_MODEL:=claude-haiku-4-5}" +# EXCEPTION 3: codex ≥0.145.0 hits the SAME rejection. For a model it has no metadata for +# (deepseek logs "Model metadata not found. Defaulting to fallback metadata") it now sends +# `reasoning:{summary:"auto"}` + `include:["reasoning.encrypted_content"]`, where 0.144.6 +# sent `reasoning:null` + `include:[]`. The gateway answers 400 "Encrypted content is not +# supported with this model" (param: include), codex exits before its first tool call, and +# BOTH probes report INCONCLUSIVE — enforcement is fine, there is just nothing to observe. +# No config override strips it (`model_reasoning_summary=none`, +# `model_supports_reasoning_summaries=false`, `model_reasoning_effort=none` all still emit +# `include`), and the old escape hatch is gone — `wire_api="chat"` is rejected outright in +# 0.145.0 (openai/codex#7782). So codex needs a model that accepts encrypted reasoning +# content. gpt-5.1-codex-mini is the cheapest that does AND supports codex's full toolset: +# gpt-5.4-nano accepts the reasoning params but 400s on `tool_search`. +# Do NOT "fix" this by pinning codex to a Claude model the way pi and claude are: the +# gateway routes Anthropic weighted 1:1 through Bedrock, which 400s on codex's request +# metadata (#576). That fails on roughly half of requests — a coin-flip red is worse in a +# daily canary than a consistent one, and a single green probe does not disprove it. +: "${CANARY_CODEX_MODEL:=gpt-5.1-codex-mini}" # Gateway base URL — overridable via env (CI supplies it as a secret). Strip any # trailing slash so the `$GW/v1` joins below never produce `//v1`. GW="${CANARY_LLM_BASE_URL:-https://models.aikin.club}"; GW="${GW%/}" @@ -129,7 +148,7 @@ drive() { # $1 = prompt ; run ONE prompt headless, executing tools without appro codex) ( cd "$BASE" && codex exec --skip-git-repo-check --dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust \ -c model_providers.gw.name="gw" -c model_providers.gw.base_url="$GW/v1" -c model_providers.gw.wire_api="responses" \ -c model_providers.gw.env_key="CANARY_LLM_API_KEY" -c model_provider="gw" \ - -c model="${CANARY_CODEX_MODEL:-deepseek-v4-pro}" "$1" 2>&1 ) ;; # Claude models 400 on Bedrock metadata; deepseek is non-Bedrock + -c model="$CANARY_CODEX_MODEL" "$1" 2>&1 ) ;; cursor) ( cd "$BASE" && cursor-agent -p --force "$1" 2>&1 ) ;; copilot) ( cd "$BASE" && copilot -p "$1" --allow-all-tools 2>&1 ) ;; devin) ( cd "$BASE" && devin -p "$1" --permission-mode dangerous --respect-workspace-trust false 2>&1 ) ;; @@ -163,7 +182,21 @@ read_denied() { grep -qE "result=deny policy=(failproofai/|custom/)?(canary-read # Vendor quota / auth errors (Copilot-Free credits, antigravity Google quota, # expired logins) → the CLI errors before any tool call. Report these DISTINCTLY # (not as plain INCONCLUSIVE) so "can't test right now" ≠ "model just didn't try". -is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b"; } +# Payload rejections belong in the same bucket: when a CLI update starts sending a param +# the pinned model refuses, the run is untestable, NOT a model that declined to act. +# Leaving those as INCONCLUSIVE is how codex 0.145.0's `include: +# ["reasoning.encrypted_content"]` 400 read as a quiet 🟡 for a full day instead of the +# ⚠️ it was. +# +# These patterns MUST stay machine-shaped. `$1` is the agent's whole transcript, so a bare +# `400` or `not supported` also matches ordinary prose ("400 tests passed", "that flag is +# not supported") and would report a chatty refusal as a vendor outage — the exact +# inversion this function exists to prevent. Hence: the structured `"code": 400` form +# rather than a loose `400`, and `not supported with` (the gateway's own phrasing: +# "not supported with this model") rather than `not supported`. Both live failures we have +# seen carry `invalid_request_error` AND `BadRequestError` anyway, so the tight forms lose +# nothing. __tests__/integration-suite/is-error.test.ts holds the fixtures both ways. +is_error() { printf '%s' "$1" | grep -qiE "quota|rate.?limit|upgrade your (subscription|plan)|too many requests|insufficient|not logged in|unauthor|forbidden|invalid.*(key|token|credential)|payment required|\\b(401|402|429)\\b|\"code\"[[:space:]]*:[[:space:]]*\"?40[0-9]|bad.?request(error)?\\b|invalid_request_error|not supported with|unsupported (parameter|model|value)|deploymentnotfound"; } ATTEMPTS=3 # retry up to N times to absorb LLM nondeterminism (flaky tool-callers) diff --git a/integration-suite/report.js b/integration-suite/report.js index ac28eda5..335493d8 100644 --- a/integration-suite/report.js +++ b/integration-suite/report.js @@ -1,15 +1,23 @@ #!/usr/bin/env node // Build the Slack report from probe verdicts + diff against last run's state. -// Args: +// Args: [channel] [peerStatePath] // Each result: {cli, probes:{...}, version?, gated?}. Prints report to stdout // AND writes new state: clis[cli] = {probes, version}. Flags broke/recovered. +// +// On the BETA channel the report answers a different question — not "is +// enforcement broken?" but "is it about to break?" — so it compares each CLI +// against the sibling stable leg's last known state (peerStatePath). const fs = require("fs"); -const [, , resultsJson, statePath, model] = process.argv; +const [, , resultsJson, statePath, model, channel = "stable", peerStatePath = "", eligible = ""] = process.argv; const results = JSON.parse(resultsJson); +const isBeta = channel !== "stable"; let prev = { clis: {}, lastRun: null }; try { prev = JSON.parse(fs.readFileSync(statePath, "utf8")); } catch {} +let peer = { clis: {} }; +if (peerStatePath) { try { peer = JSON.parse(fs.readFileSync(peerStatePath, "utf8")); } catch {} } + const probesOf = (entry) => (entry ? (entry.probes || entry) : null); // compat old flat schema const statusOf = (probes) => { const v = Object.values(probes || {}); @@ -23,16 +31,41 @@ const emo = { green: "🟢", yellow: "🟡", red: "🔴", grey: "⚪", error: " const rank = { green: 0, grey: 1, yellow: 1, error: 1, red: 2 }; const now = new Date().toISOString().replace(/\.\d+Z$/, "Z"); -const lines = [`🧪 *failproofai CLI integration tests* · ${now} · model=${model}`]; +const title = isBeta + ? `🔮 *failproofai CLI pre-release watch* · ${now} · channel=${channel} · model=${model}` + : `🧪 *failproofai CLI integration tests* · ${now} · model=${model}`; +const lines = [title]; let worst = "green"; -const breaks = [], recoveries = []; +const breaks = [], recoveries = [], incoming = []; + +// A beta CLI counts as "incoming breakage" only when the SAME CLI is green on the +// stable leg — otherwise we would report an already-broken CLI twice — and only +// after two consecutive non-green probes, so a broken alpha that gets reverted +// before release doesn't burn anyone's attention. Strike counts live in state. +const strikesOf = (cli) => (prev.clis && prev.clis[cli] && prev.clis[cli].strikes) || 0; +const newStrikes = {}; for (const r of results) { const st = statusOf(r.probes); const prevEntry = prev.clis && prev.clis[r.cli]; const prevSt = prevEntry ? statusOf(probesOf(prevEntry)) : null; let mark = ""; - if (r.gated) { + if (isBeta) { + const peerEntry = peer.clis && peer.clis[r.cli]; + const peerSt = peerEntry ? statusOf(probesOf(peerEntry)) : null; + const strikes = st === "green" ? 0 : strikesOf(r.cli) + 1; + newStrikes[r.cli] = strikes; + if (r.gated) { + mark = ` 🔵 _gated: pre-release unchanged, last green_`; + } else if (st !== "green" && peerSt === "green") { + mark = strikes >= 2 + ? ` 🚨 *INCOMING* — green on stable, ${st === "red" ? "FAILING" : "not verifiable"} on ${channel} (${strikes} runs)` + : ` 👀 _first ${channel} miss; confirming next run_`; + if (strikes >= 2) incoming.push(r.cli); + } else if (st !== "green" && peerSt && peerSt !== "green") { + mark = ` ⏸️ _also ${peerSt} on stable — the stable leg owns this alarm_`; + } + } else if (r.gated) { mark = ` 🔵 _gated: CLI v${r.version || "?"} + failproofai ${r.fpSha || "?"} unchanged, last green_`; } else if (prevSt && prevSt !== "red" && st === "red") { mark = " ⚠️ *BROKE*"; breaks.push(r.cli); @@ -48,7 +81,19 @@ for (const r of results) { const errored = results.filter((r) => statusOf(r.probes) === "error").map((r) => r.cli); let hdr; -if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`; +if (isBeta) { + // Coverage is stated explicitly and always: only some vendors publish a + // pre-release ref, and "beta: all green" must never read as assurance for the + // CLIs that were never probed at all. The denominator is how many are ELIGIBLE + // (passed in by run.sh), not 12 — a targeted run must not imply the CLIs it + // skipped have no pre-release ref, and hardcoding the eligible list here would + // be a third copy to drift against. + const total = Number(eligible) || results.length; + const cov = `_watching ${results.length}/${total} CLIs that publish a pre-release ref (of 12 total)_`; + hdr = incoming.length + ? `🚨 *INCOMING BREAKAGE*: ${incoming.join(", ")} — broken on ${channel}, still green on stable. Fix before it ships.\n${cov}` + : `🔮 no incoming breakage detected on ${channel}\n${cov}`; +} else if (breaks.length) hdr = `🔴 *ENFORCEMENT BROKEN*: ${breaks.join(", ")} — investigate now`; // Any CLI still red dominates the header — a recovery elsewhere must not mask it. else if (worst === "red") hdr = "🔴 still broken (unchanged)"; else if (recoveries.length) hdr = `✅ recovered: ${recoveries.join(", ")}`; @@ -66,6 +111,7 @@ for (const r of results) { probes: r.probes, version: r.version || (prevEntry && prevEntry.version) || null, fpSha: r.fpSha || (prevEntry && prevEntry.fpSha) || null, + ...(isBeta ? { strikes: newStrikes[r.cli] || 0 } : {}), }; } try { fs.writeFileSync(statePath, JSON.stringify(newState, null, 2)); } diff --git a/integration-suite/run.sh b/integration-suite/run.sh index e70cf90b..fab41fcb 100644 --- a/integration-suite/run.sh +++ b/integration-suite/run.sh @@ -30,7 +30,29 @@ ENVFILE="${CANARY_ENVFILE:?CANARY_ENVFILE (docker --env-file with gateway creds) MODEL="${CANARY_LLM_MODEL:-deepseek-v4-pro}" GATED="${CANARY_VERSION_GATED-all}" -CLIS=("$@"); [ ${#CLIS[@]} -eq 0 ] && CLIS=(claude codex copilot cursor factory devin antigravity goose opencode pi hermes openclaw) +CHANNEL="${CANARY_CHANNEL:-stable}" +# Sibling leg's state file, for the cross-leg comparison (see report.js). The beta +# leg needs to know whether a CLI is green on stable: "stable green + beta not +# green" is the incoming-breakage signal, and it is the ONLY way to distinguish +# "the vendor is about to break us" from "this CLI is broken for everyone already". +PEER_STATE="${CANARY_PEER_STATE:-}" + +# The CLIs with a public pre-release ref — kept in sync with the beta refs in +# install-clis.sh (__tests__/integration-suite/channel-refs.test.ts asserts the two +# lists agree, since a silent drift here would look like coverage while probing +# nothing). Its LENGTH is also the honest denominator for the beta report: a +# targeted run (`run.sh cursor`) must not imply the CLIs it skipped have no +# pre-release ref. +BETA_CLIS=(claude codex copilot cursor goose openclaw) + +CLIS=("$@") +if [ ${#CLIS[@]} -eq 0 ]; then + if [ "$CHANNEL" = stable ]; then + CLIS=(claude codex copilot cursor factory devin antigravity goose opencode pi hermes openclaw) + else + CLIS=("${BETA_CLIS[@]}") + fi +fi # failproofai main HEAD — the second gate dimension. Prefer the value the workflow # computed; fall back to git in the checkout. @@ -72,14 +94,33 @@ for cli in "${CLIS[@]}"; do vj="$(node -e 'const st=JSON.parse(require("fs").readFileSync(process.argv[1],"utf8"));const p=st.clis[process.argv[2]];process.stdout.write(JSON.stringify({cli:process.argv[2],probes:p.probes||p,version:p.version,fpSha:p.fpSha,gated:true}))' "$STATE" "$cli")" else echo ">> probing $cli ..." >&2 - vj="$(run_probe "$cli" | sed -n 's/^VERDICT_JSON //p' | tail -1)" + # Spool to a file rather than a shell variable: a probe's output is a full agent + # transcript from up to 6 CLI invocations, and a stuck or noisy client could make it + # arbitrarily large. Only the verdict line and a 20-line tail are ever needed. + out_file="$(mktemp)" + run_probe "$cli" > "$out_file" + vj="$(sed -n 's/^VERDICT_JSON //p' "$out_file" | tail -1)" + # Echo the probe tail whenever the verdict is not a clean pass. Without this the + # ONLY thing that survived a probe was its VERDICT_JSON line, so a yellow/red run + # said WHAT broke and never WHY — and re-running told you no more, because the + # cause (a vendor error message) was discarded both times. Diagnosing codex's + # 0.145.0 regression needed a full local reproduction purely for want of these + # lines. Safe in a public log: every credential here is a registered Actions + # secret, so GitHub masks it on the way out. + case "$vj" in + *FAIL*|*INCONCLUSIVE*|*ERROR*|"") + echo "── $cli probe output (tail) ──" >&2 + tail -20 "$out_file" >&2 + echo "── end $cli ──" >&2 ;; + esac + rm -f "$out_file" [ -z "$vj" ] && vj="{\"cli\":\"$cli\",\"probes\":{}}" vj="$(node -e 'const v=JSON.parse(process.argv[1]);v.version=process.argv[2]||null;v.fpSha=process.argv[3]||null;process.stdout.write(JSON.stringify(v))' "$vj" "$cur_ver" "$FP_SHA")" fi results="$(node -e 'const a=JSON.parse(process.argv[1]);a.push(JSON.parse(process.argv[2]));process.stdout.write(JSON.stringify(a))' "$results" "$vj")" done -report="$(node "$HERE/report.js" "$results" "$STATE" "$MODEL")" +report="$(node "$HERE/report.js" "$results" "$STATE" "$MODEL" "$CHANNEL" "$PEER_STATE" "${#BETA_CLIS[@]}")" echo "════ report ════" >&2; printf '%s\n' "$report" >&2; echo "════════════════" >&2 printf '%s\n' "$report" # also to stdout for the workflow log / artifact @@ -94,6 +135,17 @@ else echo "(no CANARY_SLACK_WEBHOOK set — report not posted)" >&2 fi +# The beta leg is ADVISORY and must never fail the job. It probes vendor +# pre-release builds, which are broken-by-nature often enough that gating CI on +# them would train everyone to ignore a red run — and the thing it reports is +# "this WILL break on release", not "this IS broken", which is not a reason to +# stop the world. The report is emitted and posted either way, so the signal is +# never lost; escalation is the report's job, not the exit code's. +if [ "$CHANNEL" != stable ]; then + echo "(channel=$CHANNEL — advisory only, not failing the job)" >&2 + exit 0 +fi + # Fail the job when any probe reported a hard FAIL (broken enforcement) so this # scheduled check actually blocks regressions. ERROR (vendor quota/auth) and # INCONCLUSIVE (model didn't attempt the tool) are NOT failures — they mean diff --git a/package.json b/package.json index 85badfe8..e37b1559 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "eslint-config-next": "^16.2.9", "jsdom": "^29.1.1", "lucide-react": "^1.18.0", - "next": "^16.2.9", + "next": "^16.2.11", "react": "^19.2.4", "react-dom": "^19.2.4", "tailwind-merge": "^3.4.0", @@ -107,6 +107,7 @@ "eslint-plugin-react-hooks": "7.0.1", "vite": "8.0.16", "undici": "7.28.0", - "brace-expansion": "5.0.7" + "brace-expansion": "5.0.7", + "sharp": "0.35.0" } }