From bee2ce6397e1c3414577f6c781fbbf3b5f53f72f Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 12:10:46 -0800 Subject: [PATCH 01/18] Check every listed model daily through the clients each harness ships The model picker lists each provider's models live, but each harness image pins its model client. A provider refusing a new model to an older client (Claude Code 2.1.259 for Claude Opus 5.5 and Fable 5.2) went unnoticed because nothing calls today's models through the published images. A scheduled workflow runs scripts/model-compat.ts once per published harness: it lists each provider's models as the picker does, starts the harness image (the published latest and one built from source) on each provider with a key, sends one run per model, and checks the ChatGPT plan's Codex client version against Codex's catalogue. Failures fail the run and open or update one issue; a provider without a key is reported as skipped. --- .github/workflows/model-compat.yml | 161 +++++ docs/releasing.md | 51 ++ scripts/model-compat.test.ts | 353 ++++++++++ scripts/model-compat.ts | 1054 ++++++++++++++++++++++++++++ 4 files changed, 1619 insertions(+) create mode 100644 .github/workflows/model-compat.yml create mode 100644 scripts/model-compat.test.ts create mode 100644 scripts/model-compat.ts diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml new file mode 100644 index 0000000000..c5cda7f595 --- /dev/null +++ b/.github/workflows/model-compat.yml @@ -0,0 +1,161 @@ +name: Model compatibility + +# Does every model the picker offers still answer through the clients OpenBot ships? The picker +# lists each provider's models live, but every harness image carries its model client at a fixed +# version (the Claude Agent SDK harness bundles a Claude Code CLI, the LangGraph harness sends the +# ChatGPT plan's backend a fixed Codex client version, the others carry the provider SDKs they were +# built with), and a provider can refuse a new model to an older client. scripts/model-compat.ts +# starts each harness image, one job per harness, and sends it one request per listed model on +# every provider that has a key here. See docs/releasing.md, "Model compatibility". +# +# Keys are repository secrets, one per provider. A provider without one is reported as skipped, +# not failed. On any failure the run fails and opens (or updates) one issue titled "Model +# compatibility check failing"; a passing run closes it. +# +# Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever +# runs with these keys. +on: + schedule: + - cron: "41 6 * * *" + workflow_dispatch: + inputs: + tags: + description: Harness images to check, comma-separated. `latest` is the published release people run; `source` is built from this commit, which is what the next release ships. + type: string + default: latest,source + required: false + +permissions: {} + +concurrency: + group: model-compat + cancel-in-progress: false + +jobs: + harnesses: + name: List the published harnesses + if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + images: ${{ steps.list.outputs.images }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - id: list + run: echo "images=$(jq -c . .github/published-images.json)" >> "$GITHUB_OUTPUT" + + check: + name: ${{ matrix.harness }} + needs: harnesses + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.harnesses.outputs.images) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + # Writes its results and exits 0 whatever they say: the report job below decides. + - name: One request per model through this harness + env: + HARNESS: ${{ matrix.harness }} + TAGS: ${{ inputs.tags || 'latest,source' }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + UNBIASED_API_KEY: ${{ secrets.UNBIASED_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero + --out "$RUNNER_TEMP/model-compat-$HARNESS.json" + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-compat-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat-${{ matrix.harness }}.json + if-no-files-found: error + retention-days: 14 + + report: + name: Report + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write # open, update or close the one "Model compatibility check failing" issue + env: + GH_TOKEN: ${{ github.token }} + ISSUE_TITLE: Model compatibility check failing + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Summarize + id: summary + run: | + set +e + shopt -s nullglob + bun scripts/model-compat.ts report --summary "$RUNNER_TEMP/summary.md" \ + --issue "$RUNNER_TEMP/issue-table.md" "$RUNNER_TEMP"/results/*.json + code=$? + echo "code=$code" >> "$GITHUB_OUTPUT" + if [ -f "$RUNNER_TEMP/summary.md" ]; then cat "$RUNNER_TEMP/summary.md" >> "$GITHUB_STEP_SUMMARY"; fi + exit 0 + - name: Open or update the issue + if: steps.summary.outputs.code != '0' + run: | + set -euo pipefail + body="$RUNNER_TEMP/issue.md" + { + echo "The model compatibility check failed on $(date -u +%Y-%m-%d): $RUN_URL" + echo + echo "A model refused by a pinned client means that client needs bumping: see docs/releasing.md, \"Model compatibility\". This issue is updated by every failing run and closed by the first passing one." + echo + if [ -f "$RUNNER_TEMP/issue-table.md" ]; then + cat "$RUNNER_TEMP/issue-table.md" + else + echo "No harness job wrote results. See the run." + fi + } > "$body" + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh issue create --repo "$GITHUB_REPOSITORY" --title "$ISSUE_TITLE" --body-file "$body" + fi + - name: Close the issue when everything passes + if: steps.summary.outputs.code == '0' + run: | + set -euo pipefail + number=$(gh issue list --repo "$GITHUB_REPOSITORY" --state open --search "\"$ISSUE_TITLE\" in:title" \ + --json number,title --jq "map(select(.title == env.ISSUE_TITLE)) | .[0].number // empty") + if [ -n "$number" ]; then + gh issue close "$number" --repo "$GITHUB_REPOSITORY" --comment "Passing again: $RUN_URL" + fi + - name: Fail when a model failed + if: steps.summary.outputs.code != '0' + run: | + echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." + exit 1 diff --git a/docs/releasing.md b/docs/releasing.md index 302f53223c..33765b4cf7 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1070,6 +1070,57 @@ it when it is not. Signed desktop builds are the exception: they read the `macos `windows-signing` environments described above and in [Windows signing](windows-signing.md). Downloads sync signs in to AWS with GitHub's OIDC token rather than a stored key. +## Model compatibility + +The model picker lists each provider's models live, but every harness image carries its model +client at a fixed version: the Claude Agent SDK harness bundles a Claude Code CLI (through +`claude-agent-sdk` in `agent-claude-sdk/requirements.txt`), the LangGraph harness sends the ChatGPT +plan's backend a fixed Codex client version (`DEFAULT_CLIENT_VERSION` in +`agent-langgraph-agui/src/chatgpt_models.py`), and the others carry the provider SDKs they were built +with. A provider can refuse a new model to an older client while the picker keeps offering it: +"Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required". + +**Model compatibility** (`.github/workflows/model-compat.yml`) checks for that daily, and by hand +from the Actions tab. One job per harness in `.github/published-images.json` runs +[`scripts/model-compat.ts`](../scripts/model-compat.ts), which: + +- lists each provider's models from the same source, with the same filter, as the picker (see + [The model list](configuration.md#the-model-list)); +- starts the harness image on each provider the way the Desktop configures it, and sends one AG-UI + run per model, so the harness's own client makes the call. It checks `latest`, the published + image people run, and `source`, the image built from this commit; +- compares the LangGraph harness's Codex client version with the minimum each ChatGPT plan model + needs, from openai/codex's published model catalogue. A plan sign-in cannot run in CI. A Claude + plan runs through the same Claude Code CLI as an Anthropic key, so the key's rows cover it. + +It writes a pass, fail or skipped row per harness, image, provider and model, with the provider's +own words, to the run's summary. On any failure the run fails and opens or updates one issue, +**Model compatibility check failing**, and the first passing run closes it. It runs only on a +schedule or by hand on `main`, never on a pull request. + +Each provider needs its key as a repository secret: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, +`GEMINI_API_KEY` (Google, also used for Antigravity), `XAI_API_KEY` and `UNBIASED_API_KEY`. A provider +without one is reported as skipped, not failed, and the summary names the missing secrets. + +**When it fails**, read the provider's words in the table: + +- A version refusal (a model "not supported" by the client's version, or a ChatGPT plan model that + needs a newer Codex) means bumping the pinned client: `claude-agent-sdk` in + `agent-claude-sdk/requirements.txt` (each release bundles a newer Claude Code CLI), or + `DEFAULT_CLIENT_VERSION` in `agent-langgraph-agui/src/chatgpt_models.py`, or the harness's + provider SDK in its `requirements.txt` or `package.json`. Run the check by hand: the `source` rows + show whether the bump fixed it before a release ships it, and the `latest` rows stay red until + one does. +- Billing, rate limits or a refused key are the key's problem, not a client's: fix the key. + +To run it locally (needs Docker), with keys in the environment or an env file: + +```sh +bun scripts/model-compat.ts --env-file .env --harness agent-claude-sdk --provider anthropic +``` + +`--model ` narrows the models, `--tags latest` skips building from source. + ## The one thing CI cannot do The smoke journey in `tests/smoke` is the only check that proves the parts are wired to each other: diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts new file mode 100644 index 0000000000..7a748ffc80 --- /dev/null +++ b/scripts/model-compat.test.ts @@ -0,0 +1,353 @@ +import { expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + buildMatrix, + classifyRun, + codexGate, + type Harness, + type MatrixRow, + PROVIDERS, + parseCapabilities, + providersFor, + renderTable, + usableModels, +} from "./model-compat"; + +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T12:00:00Z"); + +// The listings below are cut from what each provider answered on 2026-11-17. +const openaiListing = { + data: [ + { id: "gpt-6.1-terra", created: 1792000000 }, + { id: "gpt-6.2-sol", created: 1793000000 }, + { id: "gpt-5.5", created: 1776900000 }, + { id: "gpt-5.5-2026-04-23", created: 1776900000 }, + { id: "gpt-5.3-codex", created: 1770000000 }, + { id: "gpt-5-search-api", created: 1760000000 }, + { id: "chat-latest", created: 1760000000 }, + { id: "gpt-4o-mini-tts", created: 1740000000 }, + { id: "gpt-realtime-2", created: 1780000000 }, + { id: "gpt-image-2", created: 1776000000 }, + { id: "text-embedding-3-large", created: 1700000000 }, + { id: "whisper-1", created: 1680000000 }, + { id: "gpt-live-1", created: 1790000000 }, + { id: "gpt-4-turbo", created: 1700000000 }, + { id: "o1", created: 1720000000 }, + { id: "o3", created: 1744000000, shutdown_date: "2026-11-01" }, + { id: "o3-pro", created: 1749000000, shutdown_date: "2026-12-11" }, + ], +}; + +const anthropicListing = { + data: [ + { + id: "claude-fable-5-2", + created_at: "2026-11-09T18:30:00Z", + lifecycle: "active", + }, + { + id: "claude-opus-5-5", + created_at: "2026-09-21T16:24:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-5-5", + created_at: "2026-09-28T00:00:00Z", + lifecycle: "active", + }, + { + id: "claude-sonnet-4-5-20250929", + created_at: "2025-09-29T00:00:00Z", + retires_at: "2026-11-30T17:00:00Z", + lifecycle: "deprecated", + }, + { + id: "claude-3-opus-20240229", + created_at: "2024-02-29T00:00:00Z", + retires_at: "2026-01-05T00:00:00Z", + lifecycle: "retired", + }, + ], +}; + +const geminiListing = { + object: "list", + data: [ + { id: "models/gemini-3.8-flash", object: "model", owned_by: "google" }, + { id: "models/gemini-4-argon", object: "model", owned_by: "google" }, + { id: "models/gemini-embedding-001", object: "model", owned_by: "google" }, + { + id: "models/imagen-4.0-generate-001", + object: "model", + owned_by: "google", + }, + { id: "models/veo-3.0-generate-001", object: "model", owned_by: "google" }, + { id: "models/gemini-3.8-flash-live", object: "model", owned_by: "google" }, + { + id: "models/gemini-robotics-er-1.5", + object: "model", + owned_by: "google", + }, + { id: "models/aqa", object: "model", owned_by: "google" }, + { + id: "models/gemini-2.5-flash-preview-tts", + object: "model", + owned_by: "google", + }, + ], +}; + +test("an OpenAI key is offered only the chat models a Bot can run, newest first", () => { + expect(usableModels("openai", openaiListing, new Map(), NOW)).toEqual([ + "gpt-6.2-sol", + "gpt-6.1-terra", + "gpt-5.5", + "o3-pro", + ]); +}); + +test("models.dev saying a model takes no tools drops it, as the picker does", () => { + const capabilities = parseCapabilities({ + openai: { models: { "gpt-6.1-terra": { tool_call: false } } }, + }); + expect( + usableModels("openai", openaiListing, capabilities, NOW), + ).not.toContain("gpt-6.1-terra"); +}); + +test("an Anthropic key is offered every current Claude, retired ones dropped", () => { + expect(usableModels("anthropic", anthropicListing, new Map(), NOW)).toEqual([ + "claude-fable-5-2", + "claude-sonnet-5-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + ]); +}); + +test("a Google key keeps the Gemini chat models and drops media, live, robots and aqa", () => { + expect(usableModels("google", geminiListing, new Map(), NOW).sort()).toEqual([ + "gemini-3.8-flash", + "gemini-4-argon", + ]); +}); + +const harnesses: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { image: "agent-antigravity", credential: "google", port: 4214, runPath: "" }, + { + image: "agent-agno", + credential: "any-provider", + port: 4203, + runPath: "/agui", + }, +]; + +test("each harness is paired with the providers it can run on", () => { + expect(providersFor(harnesses[0]!)).toEqual(["anthropic"]); + expect(providersFor(harnesses[1]!)).toEqual(["google"]); + expect(providersFor(harnesses[2]!)).toEqual( + PROVIDERS.map((provider) => provider.id), + ); +}); + +test("the matrix has one row per harness, image tag and model, and skips a provider with no key", () => { + const rows = buildMatrix({ + harnesses, + tags: ["latest", "main"], + listings: new Map([ + ["anthropic", { models: ["claude-opus-5-5", "claude-sonnet-5-5"] }], + ["openai", { error: "401 Incorrect API key provided" }], + ]), + }); + const runs = rows.filter( + (row): row is Extract => row.kind === "run", + ); + // claude-sdk and agno on Anthropic: 2 harnesses x 2 tags x 2 models. + expect(runs).toHaveLength(8); + expect( + runs + .filter((row) => row.harness.image === "agent-claude-sdk") + .map((row) => `${row.tag} ${row.model}`), + ).toEqual([ + "latest claude-opus-5-5", + "latest claude-sonnet-5-5", + "main claude-opus-5-5", + "main claude-sonnet-5-5", + ]); + const listingFailures = rows.filter((row) => row.kind === "listing-failed"); + expect( + listingFailures.map((row) => [row.harness.image, row.provider]), + ).toEqual([["agent-agno", "openai"]]); + const skipped = rows.filter((row) => row.kind === "no-key"); + expect( + skipped.map((row) => [row.harness.image, row.provider]).sort(), + ).toEqual([ + ["agent-agno", "google"], + ["agent-agno", "unbiased"], + ["agent-agno", "xai"], + ["agent-antigravity", "google"], + ]); +}); + +// What the Claude Agent SDK harness at claude-agent-sdk==0.2.152 streamed for claude-opus-5-5. +const claudeSdkRefusal = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"CUSTOM","name":"system:status","value":{"status":"requesting"}}', + `data: {"type":"RUN_ERROR","message":"API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.","code":"OPENBOT_MODEL_REFUSED"}`, + "", +].join("\n\n"); + +// What the Agno harness streamed for claude-sonnet-5-5, and for a model Anthropic does not have. +const agnoReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"TEXT_MESSAGE_START","messageId":"m","role":"assistant"}', + 'data: {"type":"TEXT_MESSAGE_CONTENT","messageId":"m","delta":"OK"}', + 'data: {"type":"TEXT_MESSAGE_END","messageId":"m"}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', +].join("\n\n"); +const agnoNotFound = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + `data: {"type":"RUN_ERROR","rawEvent":{"event":"RunError","content":"litellm.NotFoundError: AnthropicException - b'{\\"type\\":\\"error\\",\\"error\\":{\\"type\\":\\"not_found_error\\",\\"message\\":\\"model: claude-nonexistent-9\\"}}'"}}`, +].join("\n\n"); + +test("a run the provider refused fails with the provider's own words", () => { + const result = classifyRun({ status: 200, body: claudeSdkRefusal }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain( + "Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required", + ); +}); + +test("a run that answered passes", () => { + expect(classifyRun({ status: 200, body: agnoReply })).toEqual({ + status: "pass", + detail: "OK", + }); +}); + +test("an error a framework puts in its raw event is still the failure's text", () => { + const result = classifyRun({ status: 200, body: agnoNotFound }); + expect(result.status).toBe("fail"); + expect(result.detail).toContain("not_found_error"); +}); + +test("a run that finished without a reply, or answered with an error as text, fails", () => { + expect( + classifyRun({ + status: 200, + body: 'data: {"type":"RUN_STARTED"}\n\ndata: {"type":"RUN_FINISHED"}', + }).status, + ).toBe("fail"); + const asText = classifyRun({ + status: 200, + body: [ + 'data: {"type":"TEXT_MESSAGE_CONTENT","delta":"Error code: 400 - model not supported"}', + 'data: {"type":"RUN_FINISHED"}', + ].join("\n\n"), + }); + expect(asText.status).toBe("fail"); + expect(asText.detail).toContain("model not supported"); +}); + +test("a refused request fails with its status and body", () => { + expect( + classifyRun({ status: 401, body: '{"error":"unauthorised"}' }), + ).toEqual({ + status: "fail", + detail: 'HTTP 401: {"error":"unauthorised"}', + }); +}); + +// Shaped as openai/codex publishes codex-rs/models-manager/models.json. +const codexCatalogue = { + models: [ + { + slug: "gpt-6.1-terra", + visibility: "list", + minimal_client_version: "0.153.0", + }, + { + slug: "gpt-6.3-sol", + visibility: "list", + minimal_client_version: "0.170.0", + }, + { + slug: "gpt-daybreak-blue-latest", + visibility: "hide", + minimal_client_version: "0.142.2", + }, + ], +}; + +test("a ChatGPT plan model newer than the harness's Codex client version fails the gate", () => { + expect(codexGate("0.167.0", codexCatalogue)).toEqual([ + { + model: "gpt-6.1-terra", + status: "pass", + detail: "needs Codex 0.153.0, harness sends 0.167.0", + }, + { + model: "gpt-6.3-sol", + status: "fail", + detail: "needs Codex 0.170.0, harness sends 0.167.0", + }, + ]); + // Compared as numbers, not text. + expect( + codexGate("0.99.0", { + models: [ + { slug: "m", visibility: "list", minimal_client_version: "0.100.0" }, + ], + })[0]!.status, + ).toBe("fail"); +}); + +test("the table escapes what would break a Markdown row", () => { + const table = renderTable([ + { + harness: "agent-agno", + tag: "latest", + provider: "openai", + model: "gpt-6.1-terra", + status: "fail", + detail: "a | b\nc", + }, + ]); + expect(table).toContain("a \\| b c"); +}); + +test("every provider the Desktop offers by address is checked", () => { + const picker = readFileSync( + join(ROOT, "desktop/src/ProviderPicker.tsx"), + "utf8", + ); + const presets = [...picker.matchAll(/baseUrl: "(https:[^"]+)"/g)].map( + (match) => match[1]!.replace(/\/+$/, ""), + ); + expect(presets.length).toBeGreaterThan(0); + const checked = PROVIDERS.map((provider) => + provider.baseUrl?.replace(/\/+$/, ""), + ); + for (const preset of presets) expect(checked).toContain(preset); +}); + +test("every published harness is one the matrix can run", () => { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses: known } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + for (const image of published) { + const harness = known.find((row) => row.image === image); + expect(harness, image).toBeDefined(); + expect(providersFor(harness!).length, image).toBeGreaterThan(0); + } +}); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts new file mode 100644 index 0000000000..fcb82b9fef --- /dev/null +++ b/scripts/model-compat.ts @@ -0,0 +1,1054 @@ +/** + * Does every model the picker offers still answer through the clients OpenBot ships? + * + * A harness image carries its own model client at a fixed version: the Claude Agent SDK harness + * bundles a Claude Code CLI, the LangGraph harness sends the ChatGPT plan's Codex backend a fixed + * client version, and every other harness carries the provider SDKs (LiteLLM, LangChain, the AI + * SDK, ...) it was built with. A provider can start refusing a model to an older client ("Claude + * Code 2.1.259 does not support this model; version 2.1.290 or newer is required") while the picker, + * which lists models live, keeps offering it. Nothing else calls today's models through the + * published images, so this does, once per model: + * + * 1. Lists each provider's models from the same source, with the same filter, as the picker + * (docs/configuration.md, "The model list"). + * 2. Starts each harness image on that provider, as the Desktop's settings would, and sends it + * one AG-UI run per model. The harness's own client makes the call, at the version that image + * ships. `latest` is the published image people run (ghcr.io/copilotkit/openbot-); + * `source` is the image built from this checkout, which shows whether a bumped pin fixed it. + * 3. Checks the ChatGPT plan's Codex client version in each image against the minimum each + * listed Codex model needs, from openai/codex's published model catalogue. A plan sign-in + * cannot run here, and the plan's version gate is a fact about the client version. + * + * A Claude plan runs through the same bundled CLI as an Anthropic key, so the key's rows cover it. + * + * Keys come from the environment (or --env-file), one per provider: ANTHROPIC_API_KEY, + * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY, UNBIASED_API_KEY. A provider with no key is + * reported as skipped, not failed. Keys are never printed: every detail is scrubbed of them. + * + * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] + * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] + * [--out results.json] [--exit-zero] + * bun scripts/model-compat.ts report --summary [--issue ] results/*.json + * + * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of + * several runs, writes the Markdown summary, and exits 1 when any of them failed. + */ + +import { randomBytes } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const ROOT = join(import.meta.dir, ".."); +const REGISTRY = "ghcr.io/copilotkit/openbot-"; +const SOURCE = "source"; +const MODELS_DEV = "https://models.dev/api.json"; +const CODEX_CATALOGUE = + "https://raw.githubusercontent.com/openai/codex/main/codex-rs/models-manager/models.json"; +const PROMPT = "Reply with the single word OK."; +const RUN_TIMEOUT_MS = 300_000; +const START_TIMEOUT_MS = 120_000; + +// --------------------------------------------------------------------------------------------- +// The picker's list. These rules are the ones the server's model catalogue applies, so this offers +// what a person can pick: the provider's own listing, minus what no Bot can chat on. +// --------------------------------------------------------------------------------------------- + +export type Source = "openai" | "anthropic" | "google" | "xai" | "gateway"; + +/** Models no chat can run on, by any provider or gateway's naming. */ +const NEVER_CHAT = + /(embed|tts|whisper|transcribe|dall-e|image|moderation|realtime|audio|sora|translate|video|lyria|imagen|veo-|(^|[-/])live([-.]|$))/i; +/** Chat models in OpenAI's listing a Bot is still not put on. */ +const OPENAI_NOT_FOR_BOTS = + /(search|deep-research|computer-use|chat-latest|codex|instruct|^gpt-live|^gpt-(3\.5|4)(-|$)|^o1(-|$)|^(babbage|davinci))/i; +/** The same, for the other providers a deployment names directly. */ +const VENDOR_NOT_FOR_BOTS = + /(deep-research|computer-use|robotics|antigravity|nano-banana|multi-agent|(^|-)aqa$)/i; +/** OpenAI's dated snapshots, which repeat a model already listed under its own name. */ +const DATED_SNAPSHOT = /-(\d{4}-\d{2}-\d{2}|\d{4})$/; + +const VENDOR_FAMILIES: readonly [RegExp, string][] = [ + [/^(gpt-|o\d|chatgpt-)/i, "openai"], + [/^claude/i, "anthropic"], + [/^(gemini|gemma)/i, "google"], + [/^grok/i, "xai"], +]; + +function modelVendor(id: string): string | undefined { + const slash = id.indexOf("/"); + if (slash > 0) return id.slice(0, slash).toLowerCase().replace(/^~/, ""); + return VENDOR_FAMILIES.find(([family]) => family.test(id))?.[1]; +} + +function chatModel(id: string, officialOpenAI: boolean): boolean { + if (NEVER_CHAT.test(id)) return false; + if (!officialOpenAI) return true; + return ( + /^(gpt-|o\d)/i.test(id) && + !OPENAI_NOT_FOR_BOTS.test(id) && + !DATED_SNAPSHOT.test(id) + ); +} + +function seconds(value: unknown): number { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string") { + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed / 1000 : 0; + } + return 0; +} + +function pastDate(value: unknown, now: number): boolean { + if (typeof value !== "string" || !value) return false; + const at = Date.parse(value); + return Number.isFinite(at) && at <= now; +} + +/** What models.dev says a model can do, by `vendor/id`. */ +export type Capabilities = Map< + string, + { tools?: boolean; text?: boolean; released?: number } +>; + +export function parseCapabilities(body: unknown): Capabilities { + const found: Capabilities = new Map(); + if (!body || typeof body !== "object") return found; + for (const vendor of ["openai", "anthropic", "google", "xai"]) { + const models = (body as Record)[vendor] + ?.models; + if (!models || typeof models !== "object") continue; + for (const [id, raw] of Object.entries(models as Record)) { + if (!raw || typeof raw !== "object") continue; + const model = raw as Record; + const outputs = (model.modalities as { output?: unknown } | undefined) + ?.output; + found.set(`${vendor}/${id}`, { + ...(typeof model.tool_call === "boolean" + ? { tools: model.tool_call } + : {}), + ...(Array.isArray(outputs) ? { text: outputs.includes("text") } : {}), + ...(typeof model.release_date === "string" + ? { released: seconds(model.release_date) } + : {}), + }); + } + } + return found; +} + +/** The models the picker offers from one listing, newest first. */ +export function usableModels( + source: Source, + body: unknown, + capabilities: Capabilities, + now: number, +): string[] { + const record = + body && typeof body === "object" ? (body as Record) : {}; + const data = Array.isArray(record.data) + ? record.data + : Array.isArray(record.models) + ? record.models + : []; + const vendorOfSource = source === "gateway" ? undefined : source; + const usable: { id: string; created: number }[] = []; + for (const raw of data) { + if (!raw || typeof raw !== "object") continue; + const item = raw as Record; + const named = + typeof item.id === "string" + ? item.id + : typeof item.name === "string" + ? item.name + : ""; + const id = named.replace(/^models\//, ""); + if (!id) continue; + const outputs = item.output_modalities; + const methods = item.supportedGenerationMethods; + const notChat = + (Array.isArray(outputs) && !outputs.includes("text")) || + (Array.isArray(methods) && !methods.includes("generateContent")); + const retired = + pastDate(item.shutdown_date, now) || + pastDate(item.retires_at, now) || + item.lifecycle === "retired"; + const bare = id.split("/").at(-1) ?? id; + const vendor = vendorOfSource ?? modelVendor(id); + const known = vendor ? capabilities.get(`${vendor}/${bare}`) : undefined; + const chat = !notChat && known?.tools !== false && known?.text !== false; + const offered = + !retired && + chat && + (source === "openai" + ? chatModel(id, true) + : source === "gateway" + ? chatModel(id, false) + : !NEVER_CHAT.test(id) && !VENDOR_NOT_FOR_BOTS.test(id)); + if (offered) + usable.push({ + id, + created: + seconds(item.created ?? item.created_at) || known?.released || 0, + }); + } + return usable + .sort((a, b) => b.created - a.created || a.id.localeCompare(b.id)) + .map((entry) => entry.id); +} + +// --------------------------------------------------------------------------------------------- +// Providers and harnesses. +// --------------------------------------------------------------------------------------------- + +/** A row of shared/harnesses.json, the fields this reads. */ +export type Harness = { + image: string | null; + credential: string; + port: number | null; + runPath: string; +}; + +export type ProviderId = "openai" | "anthropic" | "google" | "xai" | "unbiased"; + +export type Provider = { + id: ProviderId; + label: string; + /** The repository secret, and the environment variable, that holds this provider's key. */ + secret: string; + source: Source; + /** The OpenAI-compatible address the Desktop writes as OPENAI_BASE_URL for this provider. */ + baseUrl?: string; + listing: (key: string) => { url: string; headers: Record }; + /** The environment a harness is started with on this provider, as the Desktop writes it. */ + harnessEnv: (key: string, harness: Harness) => Record; +}; + +function compatible(baseUrl: string): Pick { + return { + baseUrl, + harnessEnv: (key) => ({ + OPENAI_API_KEY: key, + OPENAI_BASE_URL: baseUrl, + BOT_PROVIDER: "openai", + }), + }; +} + +function listedAt(url: string) { + return (key: string) => ({ + url, + headers: { authorization: `Bearer ${key}` }, + }); +} + +export const PROVIDERS: readonly Provider[] = [ + { + id: "openai", + label: "OpenAI", + secret: "OPENAI_API_KEY", + source: "openai", + listing: listedAt("https://api.openai.com/v1/models"), + harnessEnv: (key) => ({ OPENAI_API_KEY: key, BOT_PROVIDER: "openai" }), + }, + { + id: "anthropic", + label: "Anthropic", + secret: "ANTHROPIC_API_KEY", + source: "anthropic", + listing: (key) => ({ + url: "https://api.anthropic.com/v1/models?limit=1000", + headers: { "x-api-key": key, "anthropic-version": "2023-06-01" }, + }), + harnessEnv: (key) => ({ + ANTHROPIC_API_KEY: key, + BOT_PROVIDER: "anthropic", + }), + }, + { + id: "google", + label: "Google", + secret: "GEMINI_API_KEY", + source: "google", + listing: listedAt( + "https://generativelanguage.googleapis.com/v1beta/openai/models", + ), + ...compatible("https://generativelanguage.googleapis.com/v1beta/openai/"), + // Antigravity is pinned to Gemini's own API and reads the key under Google's name. + harnessEnv: (key, harness) => + harness.credential === "google" + ? { GOOGLE_API_KEY: key } + : compatible( + "https://generativelanguage.googleapis.com/v1beta/openai/", + ).harnessEnv(key, harness), + }, + { + id: "xai", + label: "xAI", + secret: "XAI_API_KEY", + source: "xai", + listing: listedAt("https://api.x.ai/v1/language-models"), + ...compatible("https://api.x.ai/v1"), + }, + { + id: "unbiased", + label: "Unbiased AI", + secret: "UNBIASED_API_KEY", + source: "gateway", + listing: listedAt("https://api.unbiased.ai/v1/models"), + ...compatible("https://api.unbiased.ai/v1"), + }, +]; + +function provider(id: string): Provider { + const found = PROVIDERS.find((row) => row.id === id); + if (!found) throw new Error(`No provider ${id}`); + return found; +} + +/** The providers a harness can run on, by the credential shared/harnesses.json gives it. */ +export function providersFor(harness: Harness): ProviderId[] { + if (!harness.image || !harness.port) return []; + switch (harness.credential) { + case "anthropic": + return ["anthropic"]; + case "google": + return ["google"]; + case "any-provider": + return PROVIDERS.map((row) => row.id); + default: + return []; + } +} + +export type Listing = { models: string[] } | { error: string }; + +export type MatrixRow = + | { + kind: "run"; + harness: Harness; + tag: string; + provider: ProviderId; + model: string; + } + | { kind: "no-key"; harness: Harness; provider: ProviderId } + | { + kind: "listing-failed"; + harness: Harness; + provider: ProviderId; + detail: string; + }; + +/** + * One run per harness, image tag and listed model. A provider with no key is one skipped row per + * harness, and a listing that failed one failed row, since there are no models to name. + */ +export function buildMatrix(options: { + harnesses: readonly Harness[]; + tags: readonly string[]; + listings: ReadonlyMap; +}): MatrixRow[] { + const rows: MatrixRow[] = []; + for (const harness of options.harnesses) { + for (const id of providersFor(harness)) { + const listing = options.listings.get(id); + if (!listing) rows.push({ kind: "no-key", harness, provider: id }); + else if ("error" in listing) + rows.push({ + kind: "listing-failed", + harness, + provider: id, + detail: listing.error, + }); + else + for (const tag of options.tags) + for (const model of listing.models) + rows.push({ kind: "run", harness, tag, provider: id, model }); + } + } + return rows; +} + +// --------------------------------------------------------------------------------------------- +// Reading a run. +// --------------------------------------------------------------------------------------------- + +export type Outcome = { status: "pass" | "fail"; detail: string }; + +function oneLine(text: string, limit = 400): string { + const flat = text.replace(/\s+/g, " ").trim(); + return flat.length > limit ? `${flat.slice(0, limit)}...` : flat; +} + +function errorText(event: Record): string { + if (typeof event.message === "string" && event.message) return event.message; + const raw = event.rawEvent as Record | undefined; + for (const field of ["content", "error", "message"]) { + const value = raw?.[field]; + if (typeof value === "string" && value) return value; + } + return JSON.stringify(event); +} + +/** Pass when the harness streamed a reply that says OK; otherwise fail, in the provider's words. */ +export function classifyRun(response: { + status: number; + body: string; +}): Outcome { + if (response.status !== 200) + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(response.body)}`, + }; + let text = ""; + let finished = false; + for (const line of response.body.split("\n")) { + if (!line.startsWith("data:")) continue; + let event: Record; + try { + event = JSON.parse(line.slice(5)); + } catch { + continue; + } + if (event.type === "RUN_ERROR") + return { status: "fail", detail: oneLine(errorText(event)) }; + if ( + (event.type === "TEXT_MESSAGE_CONTENT" || + event.type === "TEXT_MESSAGE_CHUNK") && + typeof event.delta === "string" + ) + text += event.delta; + if (event.type === "RUN_FINISHED") finished = true; + } + if (/\bok(ay)?\b/i.test(text)) + return { status: "pass", detail: oneLine(text, 80) }; + if (text) + return { status: "fail", detail: `answered without OK: ${oneLine(text)}` }; + return { + status: "fail", + detail: finished + ? "the run finished without a reply" + : "the stream ended without finishing the run", + }; +} + +// --------------------------------------------------------------------------------------------- +// The ChatGPT plan's version gate. +// --------------------------------------------------------------------------------------------- + +function versionParts(version: string): number[] { + return version.split(/[.-]/).map((part) => Number.parseInt(part, 10) || 0); +} + +export function compareVersions(a: string, b: string): number { + const left = versionParts(a); + const right = versionParts(b); + for (let i = 0; i < Math.max(left.length, right.length); i++) { + const difference = (left[i] ?? 0) - (right[i] ?? 0); + if (difference) return Math.sign(difference); + } + return 0; +} + +/** Each model the Codex picker lists, against the client version the harness sends the plan. */ +export function codexGate( + clientVersion: string, + catalogue: unknown, +): { model: string; status: "pass" | "fail"; detail: string }[] { + const models = (catalogue as { models?: unknown })?.models; + if (!Array.isArray(models)) return []; + return models + .filter( + (row): row is { slug: string; minimal_client_version?: string } => + !!row && row.visibility === "list" && typeof row.slug === "string", + ) + .map((row) => { + const needs = row.minimal_client_version ?? "0.0.0"; + return { + model: row.slug, + status: compareVersions(clientVersion, needs) >= 0 ? "pass" : "fail", + detail: `needs Codex ${needs}, harness sends ${clientVersion}`, + }; + }); +} + +// --------------------------------------------------------------------------------------------- +// Results. +// --------------------------------------------------------------------------------------------- + +export type Result = { + harness: string; + tag: string; + provider: string; + model: string; + status: "pass" | "fail" | "skipped"; + detail: string; +}; + +function cell(text: string): string { + return oneLine(text).replace(/\|/g, "\\|"); +} + +export function renderTable(results: readonly Result[]): string { + const lines = [ + "| Harness | Image | Provider | Model | Result | Provider's answer |", + "| --- | --- | --- | --- | --- | --- |", + ]; + for (const row of results) + lines.push( + `| ${cell(row.harness)} | ${cell(row.tag)} | ${cell(row.provider)} | ${cell(row.model)} | ${row.status} | ${cell(row.detail)} |`, + ); + return lines.join("\n"); +} + +const ORDER = { fail: 0, skipped: 1, pass: 2 } as const; + +/** + * The summary: what failed first, then what was skipped for want of a key, then what passed. The + * issue's copy leaves out what passed and keeps at most `maxFailed` failures, since an issue body + * holds 65,536 characters and the run's own summary has the rest. + */ +export function renderSummary( + results: readonly Result[], + options: { passed?: boolean; maxFailed?: number } = {}, +): string { + const { passed: showPassed = true, maxFailed = Number.POSITIVE_INFINITY } = + options; + const sorted = [...results].sort( + (a, b) => + ORDER[a.status] - ORDER[b.status] || + a.harness.localeCompare(b.harness) || + a.provider.localeCompare(b.provider) || + a.tag.localeCompare(b.tag) || + a.model.localeCompare(b.model), + ); + const count = (status: Result["status"]) => + results.filter((row) => row.status === status).length; + const failed = sorted.filter((row) => row.status === "fail"); + const skipped = sorted.filter((row) => row.status === "skipped"); + const passed = sorted.filter((row) => row.status === "pass"); + const missing = [ + ...new Set( + skipped.map( + (row) => + PROVIDERS.find((p) => p.label === row.provider)?.secret ?? + row.provider, + ), + ), + ]; + const parts = [ + "## Model compatibility", + "", + `${count("fail")} failed, ${count("pass")} passed, ${count("skipped")} skipped.`, + "", + ]; + if (missing.length) + parts.push( + `Skipped for want of a key, not failed: add the repository secret${missing.length > 1 ? "s" : ""} ${missing.map((name) => `\`${name}\``).join(", ")} to check ${missing.length > 1 ? "those providers" : "that provider"}.`, + "", + ); + if (failed.length) { + parts.push("### Failed", "", renderTable(failed.slice(0, maxFailed)), ""); + if (failed.length > maxFailed) + parts.push( + `And ${failed.length - maxFailed} more failures, in the run's summary.`, + "", + ); + } + if (skipped.length) parts.push("### Skipped", "", renderTable(skipped), ""); + if (passed.length && showPassed) + parts.push( + "
Passed", + "", + renderTable(passed), + "", + "
", + "", + ); + return parts.join("\n"); +} + +// --------------------------------------------------------------------------------------------- +// Running it. +// --------------------------------------------------------------------------------------------- + +type Options = { + envFile?: string; + harnesses?: string[]; + providers?: string[]; + model?: RegExp; + tags: string[]; + concurrency: number; + out?: string; + exitZero: boolean; +}; + +function parseOptions(argv: string[]): Options { + const options: Options = { + tags: ["latest", SOURCE], + concurrency: 6, + exitZero: false, + }; + const list = (value: string | undefined) => + (value ?? "") + .split(/[\s,]+/) + .map((item) => item.trim()) + .filter(Boolean); + for (let i = 0; i < argv.length; i++) { + const flag = argv[i]; + const value = () => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${flag} needs a value`); + return next; + }; + if (flag === "--env-file") options.envFile = value(); + else if (flag === "--harness") options.harnesses = list(value()); + else if (flag === "--provider") options.providers = list(value()); + else if (flag === "--model") options.model = new RegExp(value()); + else if (flag === "--tags") options.tags = list(value()); + else if (flag === "--concurrency") + options.concurrency = Math.max(1, Number(value()) || 1); + else if (flag === "--out") options.out = value(); + else if (flag === "--exit-zero") options.exitZero = true; + else throw new Error(`Unknown option ${flag}`); + } + return options; +} + +/** The provider keys, from the environment, or from an env file for the names it lacks. */ +function readKeys(envFile: string | undefined): Map { + const fromFile = new Map(); + if (envFile) + for (const line of readFileSync(envFile, "utf8").split("\n")) { + const match = /^\s*(?:export\s+)?([A-Z0-9_]+)\s*=\s*(.*)$/.exec(line); + if (!match) continue; + const [, name = "", raw = ""] = match; + fromFile.set(name, raw.trim().replace(/^(['"])(.*)\1$/, "$2")); + } + const keys = new Map(); + for (const row of PROVIDERS) { + const value = ( + process.env[row.secret] || + fromFile.get(row.secret) || + "" + ).trim(); + if (value) keys.set(row.id, value); + } + return keys; +} + +function scrubber(keys: Map) { + const secrets = [...keys.values()].filter((value) => value.length >= 8); + return (text: string) => + secrets.reduce( + (scrubbed, secret) => scrubbed.split(secret).join("[key]"), + text, + ); +} + +async function fetchJson( + url: string, + headers: Record = {}, +): Promise<{ ok: true; body: unknown } | { ok: false; error: string }> { + try { + const response = await fetch(url, { + headers, + signal: AbortSignal.timeout(30_000), + }); + const text = await response.text(); + if (!response.ok) + return { ok: false, error: `HTTP ${response.status}: ${oneLine(text)}` }; + return { ok: true, body: JSON.parse(text) }; + } catch (error) { + return { ok: false, error: String(error) }; + } +} + +async function exec( + args: string[], + env: Record = {}, +): Promise<{ code: number; stdout: string; stderr: string }> { + const child = Bun.spawn(args, { + env: { ...process.env, ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + return { code, stdout, stderr }; +} + +/** + * `source` is the image built from this checkout, which is what the next release ships, so a pin + * bumped here is checked before it is released. Any other tag is the published image: `latest` is + * what people run today. + */ +function imageOf(harness: Harness, tag: string): string { + return tag === SOURCE + ? `model-compat/${harness.image}:source` + : `${REGISTRY}${harness.image}:${tag}`; +} + +async function prepare(harness: Harness, tag: string) { + return tag === SOURCE + ? exec([ + "docker", + "build", + "--quiet", + "-f", + join(ROOT, harness.image ?? "", "Dockerfile"), + "-t", + imageOf(harness, tag), + ROOT, + ]) + : exec(["docker", "pull", "--quiet", imageOf(harness, tag)]); +} + +let serial = 0; + +/** Starts the harness on one provider and model, sends one run, and stops it. */ +async function runOne( + row: Extract, + key: string, + scrub: (text: string) => string, +): Promise { + const harness = row.harness; + const name = `model-compat-${process.pid}-${++serial}`; + const env = { + ...provider(row.provider).harnessEnv(key, harness), + BOT_MODEL: row.model, + }; + const token = randomBytes(12).toString("hex"); + // Each value goes to Docker through the environment and is named here, so no key is in argv. + const started = await exec( + [ + "docker", + "run", + "-d", + "--rm", + "--name", + name, + "-p", + `127.0.0.1::${harness.port}`, + "-e", + "MANAGED_AGENT_TOKEN", + ...Object.keys(env).flatMap((variable) => ["-e", variable]), + imageOf(harness, row.tag), + ], + { ...env, MANAGED_AGENT_TOKEN: token }, + ); + if (started.code !== 0) + return { + status: "fail", + detail: scrub(`the harness did not start: ${oneLine(started.stderr)}`), + }; + try { + const port = await exec(["docker", "port", name, `${harness.port}/tcp`]); + const address = port.stdout.trim().split("\n")[0]?.trim(); + if (!address) + return { status: "fail", detail: "the harness published no port" }; + const base = `http://${address}`; + const deadline = Date.now() + START_TIMEOUT_MS; + let healthy = false; + while (Date.now() < deadline) { + try { + const health = await fetch(`${base}/health`, { + signal: AbortSignal.timeout(5_000), + }); + if (health.ok) { + healthy = true; + break; + } + } catch { + // Not listening yet. + } + const state = await exec([ + "docker", + "inspect", + "-f", + "{{.State.Running}}", + name, + ]); + if (state.stdout.trim() !== "true") break; + await Bun.sleep(1_000); + } + if (!healthy) { + const logs = await exec(["docker", "logs", "--tail", "5", name]); + return { + status: "fail", + detail: scrub( + `the harness never became healthy: ${oneLine(logs.stdout + logs.stderr)}`, + ), + }; + } + const id = randomBytes(6).toString("hex"); + let response: { status: number; body: string }; + try { + const reply = await fetch(`${base}${harness.runPath || "/"}`, { + method: "POST", + headers: { + "content-type": "application/json", + accept: "text/event-stream", + "x-openbot-agent-token": token, + }, + body: JSON.stringify({ + threadId: `model-compat-${id}`, + runId: `run-${id}`, + state: {}, + messages: [{ id: `message-${id}`, role: "user", content: PROMPT }], + tools: [], + context: [], + // What the server tells the LangGraph and Claude Agent SDK harnesses; the others read + // BOT_MODEL, set above. + forwardedProps: { openbotModel: { model: row.model } }, + }), + signal: AbortSignal.timeout(RUN_TIMEOUT_MS), + }); + response = { status: reply.status, body: await reply.text() }; + } catch (error) { + return { + status: "fail", + detail: scrub(`no answer: ${oneLine(String(error))}`), + }; + } + const outcome = classifyRun(response); + if (outcome.status === "fail" && !response.body.includes("RUN_ERROR")) { + // A harness that swallowed the error still logged it. + const logs = await exec(["docker", "logs", "--tail", "40", name]); + const said = (logs.stdout + logs.stderr) + .split("\n") + .filter((line) => /error|exception/i.test(line)) + .at(-1); + if (said) + outcome.detail = `${outcome.detail}; logged: ${oneLine(said, 300)}`; + } + return { status: outcome.status, detail: scrub(outcome.detail) }; + } finally { + await exec(["docker", "rm", "-f", name]); + } +} + +async function pool( + items: readonly T[], + width: number, + work: (item: T) => Promise, +) { + let next = 0; + await Promise.all( + Array.from({ length: Math.min(width, items.length) }, async () => { + for (let item = items[next++]; item !== undefined; item = items[next++]) + await work(item); + }), + ); +} + +async function check(argv: string[]): Promise { + const options = parseOptions(argv); + const keys = readKeys(options.envFile); + const scrub = scrubber(keys); + + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const harnesses = ( + JSON.parse(readFileSync(join(ROOT, "shared/harnesses.json"), "utf8")) as { + harnesses: Harness[]; + } + ).harnesses.filter( + (row) => + row.image && + published.includes(row.image) && + (!options.harnesses || options.harnesses.includes(row.image)), + ); + if (!harnesses.length) + throw new Error("No published harness matches --harness."); + + const wanted = new Set( + harnesses + .flatMap(providersFor) + .filter((id) => !options.providers || options.providers.includes(id)), + ); + const capabilities = await fetchJson(MODELS_DEV); + const known = capabilities.ok + ? parseCapabilities(capabilities.body) + : new Map(); + const listings = new Map(); + for (const id of wanted) { + const key = keys.get(id); + if (!key) continue; + const row = provider(id); + const request = row.listing(key); + const listed = await fetchJson(request.url, request.headers); + listings.set( + id, + listed.ok + ? { + models: usableModels( + row.source, + listed.body, + known, + Date.now(), + ).filter((model) => !options.model || options.model.test(model)), + } + : { error: scrub(`the model listing failed: ${listed.error}`) }, + ); + } + const matrix = buildMatrix({ + harnesses, + tags: options.tags, + listings, + }).filter( + (row) => !options.providers || options.providers.includes(row.provider), + ); + + const results: Result[] = []; + const label = (id: string) => provider(id).label; + for (const row of matrix) { + if (row.kind === "no-key") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + model: "(all)", + status: "skipped", + detail: `skipped: no key (${provider(row.provider).secret})`, + }); + if (row.kind === "listing-failed") + results.push({ + harness: row.harness.image ?? "", + tag: options.tags.join(", "), + provider: label(row.provider), + model: "(all)", + status: "fail", + detail: row.detail, + }); + } + + // Every image once, before any run, so a pull or build is never counted against a run's time. + const images = harnesses.flatMap((harness) => + options.tags.map((tag) => ({ harness, tag })), + ); + const unpulled = new Map(); + await pool(images, 3, async ({ harness, tag }) => { + const ready = await prepare(harness, tag); + if (ready.code !== 0) + unpulled.set( + imageOf(harness, tag), + oneLine(ready.stderr.split("\n").slice(-5).join(" ")), + ); + }); + + // The ChatGPT plan's Codex client version, in whichever images carry the plan. + const codex = await fetchJson(CODEX_CATALOGUE); + if (!options.providers || options.providers.includes("openai")) + for (const harness of harnesses) + for (const tag of options.tags) { + const image = imageOf(harness, tag); + if (unpulled.has(image)) continue; + const version = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + "python", + image, + "-c", + "from src.chatgpt_models import client_version; print(client_version())", + ]); + if (version.code !== 0) continue; + const sends = version.stdout.trim(); + if (!codex.ok) { + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + model: "(all)", + status: "fail", + detail: `could not read Codex's model catalogue: ${codex.error}`, + }); + continue; + } + for (const gate of codexGate(sends, codex.body)) + results.push({ + harness: harness.image ?? "", + tag, + provider: "ChatGPT plan", + ...gate, + }); + } + + const runs = matrix.filter( + (row): row is Extract => row.kind === "run", + ); + let done = 0; + await pool(runs, options.concurrency, async (row) => { + const image = imageOf(row.harness, row.tag); + const outcome = unpulled.has(image) + ? { + status: "fail" as const, + detail: `could not get ${image}: ${unpulled.get(image)}`, + } + : await runOne(row, keys.get(row.provider) ?? "", scrub); + results.push({ + harness: row.harness.image ?? "", + tag: row.tag, + provider: + row.provider === "anthropic" && row.harness.credential === "anthropic" + ? "Anthropic (key or Claude plan)" + : label(row.provider), + model: row.model, + ...outcome, + }); + done++; + console.error( + `[${done}/${runs.length}] ${row.harness.image}:${row.tag} ${row.provider} ${row.model}: ${outcome.status}`, + ); + }); + + if (options.out) + writeFileSync(options.out, `${JSON.stringify({ results }, null, 2)}\n`); + console.log(renderSummary(results)); + return results.some((row) => row.status === "fail") && !options.exitZero + ? 1 + : 0; +} + +function report(argv: string[]): number { + let summary: string | undefined; + let issue: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--summary") summary = argv[++i]; + else if (arg === "--issue") issue = argv[++i]; + else if (arg) files.push(arg); + } + const results = files.flatMap( + (file) => + (JSON.parse(readFileSync(file, "utf8")) as { results: Result[] }).results, + ); + const markdown = renderSummary(results); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + if (issue) + writeFileSync( + issue, + `${renderSummary(results, { passed: false, maxFailed: 150 })}\n`, + ); + if (!results.length) { + console.error( + "No results were found: every harness job failed before it wrote any.", + ); + return 1; + } + return results.some((row) => row.status === "fail") ? 1 : 0; +} + +if (import.meta.main) { + const argv = process.argv.slice(2); + const code = argv[0] === "report" ? report(argv.slice(1)) : await check(argv); + process.exit(code); +} From 30e39ff42f86c043ef4843b6e11f10a82d455121 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 12:13:51 -0800 Subject: [PATCH 02/18] Log how long each harness image took to pull or build --- scripts/model-compat.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index fcb82b9fef..8291d1ab00 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -933,7 +933,11 @@ async function check(argv: string[]): Promise { ); const unpulled = new Map(); await pool(images, 3, async ({ harness, tag }) => { + const began = Date.now(); const ready = await prepare(harness, tag); + console.error( + `${tag === SOURCE ? "built" : "pulled"} ${imageOf(harness, tag)} in ${Math.round((Date.now() - began) / 1000)}s${ready.code ? " (failed)" : ""}`, + ); if (ready.code !== 0) unpulled.set( imageOf(harness, tag), From 33c2784bec2e29198a2897cc6a8bf722c85ec1b1 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 12:29:50 -0800 Subject: [PATCH 03/18] Read CrewAI's snapshot replies and Mastra's own API, and give every run a tool --- scripts/model-compat.test.ts | 38 +++++++++++ scripts/model-compat.ts | 128 +++++++++++++++++++++++++++-------- 2 files changed, 138 insertions(+), 28 deletions(-) diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts index 7a748ffc80..25c257b7ac 100644 --- a/scripts/model-compat.test.ts +++ b/scripts/model-compat.test.ts @@ -3,6 +3,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { buildMatrix, + classifyMastra, classifyRun, codexGate, type Harness, @@ -256,6 +257,43 @@ test("a run that finished without a reply, or answered with an error as text, fa expect(asText.detail).toContain("model not supported"); }); +test("a reply that arrives only as a messages snapshot, as CrewAI's bridge sends it, passes", () => { + const crewaiReply = [ + 'data: {"type":"RUN_STARTED","threadId":"t","runId":"r"}', + 'data: {"type":"MESSAGES_SNAPSHOT","messages":[{"id":"u","role":"user","content":"Reply with the single word OK."},{"id":"a","role":"assistant","content":"OK"}]}', + 'data: {"type":"RUN_FINISHED","threadId":"t","runId":"r"}', + ].join("\n\n"); + expect(classifyRun({ status: 200, body: crewaiReply })).toEqual({ + status: "pass", + detail: "OK", + }); + // The person's own message in the snapshot is not a reply. + const onlyThePrompt = crewaiReply.replace( + ',{"id":"a","role":"assistant","content":"OK"}', + "", + ); + expect(classifyRun({ status: 200, body: onlyThePrompt }).status).toBe("fail"); +}); + +test("Mastra's own API answer is read as a reply or a refusal", () => { + expect( + classifyMastra({ + status: 200, + body: '{"text":"OK","finishReason":"stop"}', + }), + ).toEqual({ status: "pass", detail: "OK" }); + expect( + classifyMastra({ + status: 500, + body: '{"error":"This model is only supported in v1/responses and not in v1/chat/completions."}', + }), + ).toEqual({ + status: "fail", + detail: + "HTTP 500: This model is only supported in v1/responses and not in v1/chat/completions.", + }); +}); + test("a refused request fails with its status and body", () => { expect( classifyRun({ status: 401, body: '{"error":"unauthorised"}' }), diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index 8291d1ab00..c8bbb9ceb7 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -45,6 +45,16 @@ const MODELS_DEV = "https://models.dev/api.json"; const CODEX_CATALOGUE = "https://raw.githubusercontent.com/openai/codex/main/codex-rs/models-manager/models.json"; const PROMPT = "Reply with the single word OK."; +/** One tool the prompt has no use for, so the request takes the client's tool path. */ +const PROBE_TOOL = { + name: "look_up_order", + description: "Look up the status of an order by its number.", + parameters: { + type: "object", + properties: { order: { type: "string", description: "The order number." } }, + required: ["order"], + }, +}; const RUN_TIMEOUT_MS = 300_000; const START_TIMEOUT_MS = 120_000; @@ -400,8 +410,11 @@ export function classifyRun(response: { status: "fail", detail: `HTTP ${response.status}: ${oneLine(response.body)}`, }; - let text = ""; + let streamed = ""; + // Some bridges (CrewAI's) send the reply only as the conversation's new state. + let snapshot = ""; let finished = false; + let calledTool = false; for (const line of response.body.split("\n")) { if (!line.startsWith("data:")) continue; let event: Record; @@ -417,19 +430,64 @@ export function classifyRun(response: { event.type === "TEXT_MESSAGE_CHUNK") && typeof event.delta === "string" ) - text += event.delta; + streamed += event.delta; + if (event.type === "MESSAGES_SNAPSHOT" && Array.isArray(event.messages)) { + const last = event.messages.at(-1) as + | { role?: unknown; content?: unknown } + | undefined; + snapshot = + last?.role === "assistant" && typeof last.content === "string" + ? last.content + : ""; + } + if (event.type === "TOOL_CALL_START") calledTool = true; if (event.type === "RUN_FINISHED") finished = true; } + return judgeReply( + streamed || snapshot, + calledTool + ? "the model called the probe's tool instead of replying" + : finished + ? "the run finished without a reply" + : "the stream ended without finishing the run", + ); +} + +function judgeReply(text: string, silence: string): Outcome { if (/\bok(ay)?\b/i.test(text)) return { status: "pass", detail: oneLine(text, 80) }; if (text) return { status: "fail", detail: `answered without OK: ${oneLine(text)}` }; - return { - status: "fail", - detail: finished - ? "the run finished without a reply" - : "the stream ended without finishing the run", - }; + return { status: "fail", detail: silence }; +} + +/** + * Mastra serves no AG-UI route: OpenBot reaches it through Mastra's own agent API, so its run is a + * `generate` call on that API, answered with the reply's `text` or an `error`. + */ +export function classifyMastra(response: { + status: number; + body: string; +}): Outcome { + let parsed: { text?: unknown; error?: unknown; message?: unknown } = {}; + try { + parsed = JSON.parse(response.body); + } catch { + // Not JSON: reported as it is, below. + } + if (response.status !== 200) { + const said = [parsed.error, parsed.message].find( + (value): value is string => typeof value === "string" && !!value, + ); + return { + status: "fail", + detail: `HTTP ${response.status}: ${oneLine(said ?? response.body)}`, + }; + } + return judgeReply( + typeof parsed.text === "string" ? parsed.text : "", + "the agent answered without text", + ); } // --------------------------------------------------------------------------------------------- @@ -785,28 +843,42 @@ async function runOne( }; } const id = randomBytes(6).toString("hex"); + const mastra = harness.image === "agent-mastra"; let response: { status: number; body: string }; try { - const reply = await fetch(`${base}${harness.runPath || "/"}`, { - method: "POST", - headers: { - "content-type": "application/json", - accept: "text/event-stream", - "x-openbot-agent-token": token, + const reply = await fetch( + mastra + ? `${base}/api/agents/openbot/generate` + : `${base}${harness.runPath || "/"}`, + { + method: "POST", + headers: { + "content-type": "application/json", + accept: mastra ? "application/json" : "text/event-stream", + "x-openbot-agent-token": token, + }, + body: JSON.stringify( + mastra + ? { messages: [{ role: "user", content: PROMPT }] } + : { + threadId: `model-compat-${id}`, + runId: `run-${id}`, + state: {}, + messages: [ + { id: `message-${id}`, role: "user", content: PROMPT }, + ], + // A Bot's run always carries tools, and a client's tool path is where models + // part ways (Responses-only models, parallel tool calls), so this one does too. + tools: [PROBE_TOOL], + context: [], + // What the server tells the LangGraph and Claude Agent SDK harnesses; the others + // read BOT_MODEL, set above. + forwardedProps: { openbotModel: { model: row.model } }, + }, + ), + signal: AbortSignal.timeout(RUN_TIMEOUT_MS), }, - body: JSON.stringify({ - threadId: `model-compat-${id}`, - runId: `run-${id}`, - state: {}, - messages: [{ id: `message-${id}`, role: "user", content: PROMPT }], - tools: [], - context: [], - // What the server tells the LangGraph and Claude Agent SDK harnesses; the others read - // BOT_MODEL, set above. - forwardedProps: { openbotModel: { model: row.model } }, - }), - signal: AbortSignal.timeout(RUN_TIMEOUT_MS), - }); + ); response = { status: reply.status, body: await reply.text() }; } catch (error) { return { @@ -814,7 +886,7 @@ async function runOne( detail: scrub(`no answer: ${oneLine(String(error))}`), }; } - const outcome = classifyRun(response); + const outcome = mastra ? classifyMastra(response) : classifyRun(response); if (outcome.status === "fail" && !response.body.includes("RUN_ERROR")) { // A harness that swallowed the error still logged it. const logs = await exec(["docker", "logs", "--tail", "40", name]); From d2d1d5661dcda503e32fbbbb68f9025b757dee10 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 12:40:59 -0800 Subject: [PATCH 04/18] Say what to do when a harness's request shape is refused --- docs/releasing.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/releasing.md b/docs/releasing.md index 33765b4cf7..f94d072d52 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1111,6 +1111,9 @@ without one is reported as skipped, not failed, and the summary names the missin provider SDK in its `requirements.txt` or `package.json`. Run the check by hand: the `source` rows show whether the bump fixed it before a release ships it, and the `latest` rows stay red until one does. +- A refused parameter or endpoint (a model that takes tools only through `/v1/responses`, or + refuses a `temperature`) means that harness builds its request in a way the model no longer + accepts: fix how the harness sets up its model, or bump the framework if a newer one does. - Billing, rate limits or a refused key are the key's problem, not a client's: fix the key. To run it locally (needs Docker), with keys in the environment or an env file: From e0f7b09f0c9c50436575010ec56cc96f52646ac7 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:17:10 -0800 Subject: [PATCH 05/18] Name the model check's own requests in the client boundary, and leave Unbiased AI out The client boundary test failed on scripts/model-compat.ts: its provider listings, harness health checks and Mastra's agent API are not OpenBot's server, so they are allowed as third-party. Unbiased AI is a gateway serving its own model, so the check no longer lists or runs it, and the Desktop drift test names it as left out on purpose. --- .github/workflows/model-compat.yml | 1 - api/client-boundary.allow.ts | 6 ++++++ docs/releasing.md | 5 +++-- scripts/model-compat.test.ts | 9 +++++---- scripts/model-compat.ts | 21 ++++++++++++--------- 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml index c5cda7f595..bc650d18f2 100644 --- a/.github/workflows/model-compat.yml +++ b/.github/workflows/model-compat.yml @@ -75,7 +75,6 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - UNBIASED_API_KEY: ${{ secrets.UNBIASED_API_KEY }} run: >- bun scripts/model-compat.ts --harness "$HARNESS" --tags "$TAGS" --exit-zero --out "$RUNNER_TEMP/model-compat-$HARNESS.json" diff --git a/api/client-boundary.allow.ts b/api/client-boundary.allow.ts index 7dd7aa1964..cadc9bbe89 100644 --- a/api/client-boundary.allow.ts +++ b/api/client-boundary.allow.ts @@ -398,6 +398,12 @@ export const ALLOWED: Allowed[] = [ rule: "third-party", why: "image registries, never OpenBot's server", }, + { + file: "scripts/model-compat.ts", + what: /^(fetch\(\)|\/health|\/api\/agents\/openbot\/generate)$/, + rule: "third-party", + why: "model providers' listings and the harness images the model check starts on this machine, never OpenBot's server", + }, // Scripts that run and stop a server on this machine. { file: "scripts/update-local.ts", diff --git a/docs/releasing.md b/docs/releasing.md index f94d072d52..533be73faa 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1099,8 +1099,9 @@ own words, to the run's summary. On any failure the run fails and opens or updat schedule or by hand on `main`, never on a pull request. Each provider needs its key as a repository secret: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, -`GEMINI_API_KEY` (Google, also used for Antigravity), `XAI_API_KEY` and `UNBIASED_API_KEY`. A provider -without one is reported as skipped, not failed, and the summary names the missing secrets. +`GEMINI_API_KEY` (Google, also used for Antigravity) and `XAI_API_KEY`. A provider without one is +reported as skipped, not failed, and the summary names the missing secrets. Unbiased AI is not +checked: it is a gateway serving its own model, not a provider's listing. **When it fails**, read the provider's words in the table: diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts index 25c257b7ac..c4ce089879 100644 --- a/scripts/model-compat.test.ts +++ b/scripts/model-compat.test.ts @@ -8,6 +8,7 @@ import { codexGate, type Harness, type MatrixRow, + NOT_CHECKED, PROVIDERS, parseCapabilities, providersFor, @@ -191,7 +192,6 @@ test("the matrix has one row per harness, image tag and model, and skips a provi skipped.map((row) => [row.harness.image, row.provider]).sort(), ).toEqual([ ["agent-agno", "google"], - ["agent-agno", "unbiased"], ["agent-agno", "xai"], ["agent-antigravity", "google"], ]); @@ -370,9 +370,10 @@ test("every provider the Desktop offers by address is checked", () => { (match) => match[1]!.replace(/\/+$/, ""), ); expect(presets.length).toBeGreaterThan(0); - const checked = PROVIDERS.map((provider) => - provider.baseUrl?.replace(/\/+$/, ""), - ); + const checked = [ + ...PROVIDERS.map((provider) => provider.baseUrl), + ...NOT_CHECKED.map((left) => left.baseUrl), + ].map((url) => url?.replace(/\/+$/, "")); for (const preset of presets) expect(checked).toContain(preset); }); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index c8bbb9ceb7..fa4bd8ad47 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -22,8 +22,8 @@ * A Claude plan runs through the same bundled CLI as an Anthropic key, so the key's rows cover it. * * Keys come from the environment (or --env-file), one per provider: ANTHROPIC_API_KEY, - * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY, UNBIASED_API_KEY. A provider with no key is - * reported as skipped, not failed. Keys are never printed: every detail is scrubbed of them. + * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY. A provider with no key is reported as skipped, + * not failed. Keys are never printed: every detail is scrubbed of them. * * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] @@ -219,7 +219,7 @@ export type Harness = { runPath: string; }; -export type ProviderId = "openai" | "anthropic" | "google" | "xai" | "unbiased"; +export type ProviderId = "openai" | "anthropic" | "google" | "xai"; export type Provider = { id: ProviderId; @@ -300,13 +300,16 @@ export const PROVIDERS: readonly Provider[] = [ listing: listedAt("https://api.x.ai/v1/language-models"), ...compatible("https://api.x.ai/v1"), }, +]; + +/** + * Providers the Desktop offers by address that this check leaves out on purpose, with why. A + * gateway serves its own aliases, not a provider's models. + */ +export const NOT_CHECKED: readonly { baseUrl: string; why: string }[] = [ { - id: "unbiased", - label: "Unbiased AI", - secret: "UNBIASED_API_KEY", - source: "gateway", - listing: listedAt("https://api.unbiased.ai/v1/models"), - ...compatible("https://api.unbiased.ai/v1"), + baseUrl: "https://api.unbiased.ai/v1", + why: "Unbiased AI is a gateway serving its own `pareto` model, not a provider's listing", }, ]; From 0b3bc66b95be4b0ede8c9946d571be8a208c9f7e Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:17:20 -0800 Subject: [PATCH 06/18] Offer a model only when the model check has proven it on the Bot's harness Adds the model lock: shared/model-lock.json, written from the model check's source-image results by scripts/model-compat.ts lock, and the rule in shared/model-lock.ts that the model list applies. The daily check opens or updates one pull request when a model starts or stops answering; a pull request that changes a harness's client runs the check for that harness and fails when a verified model breaks. A conversation set to a model that isn't verified says so in the composer (ModelChoice.unverified). --- .github/workflows/model-compat.yml | 96 ++ .github/workflows/model-lock.yml | 103 ++ api/openapi.yaml | 15 + api/openapi/model-connections.yaml | 17 + app/src/lib/api/generated/model/index.ts | 1 + .../lib/api/generated/model/modelChoice.ts | 2 + .../api/generated/model/unverifiedModel.ts | 13 + .../lib/api/generated/models/models.zod.ts | 18 + app/src/lib/models/queries.ts | 6 + app/src/lib/models/run-model.ts | 24 +- docs/releasing.md | 31 + scripts/model-compat.ts | 403 ++++- scripts/model-lock.test.ts | 232 +++ .../model-listings/anthropic-models.json | 1327 +++++++++++++++++ .../model-listings/openai-models.json | 705 +++++++++ shared/model-lock.json | 4 + shared/model-lock.test.ts | 90 ++ shared/model-lock.ts | 94 ++ 18 files changed, 3174 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/model-lock.yml create mode 100644 app/src/lib/api/generated/model/unverifiedModel.ts create mode 100644 scripts/model-lock.test.ts create mode 100644 shared/fixtures/model-listings/anthropic-models.json create mode 100644 shared/fixtures/model-listings/openai-models.json create mode 100644 shared/model-lock.json create mode 100644 shared/model-lock.test.ts create mode 100644 shared/model-lock.ts diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml index bc650d18f2..f41029ae1f 100644 --- a/.github/workflows/model-compat.yml +++ b/.github/workflows/model-compat.yml @@ -12,6 +12,10 @@ name: Model compatibility # not failed. On any failure the run fails and opens (or updates) one issue titled "Model # compatibility check failing"; a passing run closes it. # +# What answered on the `source` image is the model lock (shared/model-lock.json), the models the +# picker may offer for each harness. When a model newly answers or stops answering, the run opens +# or updates one pull request that refreshes the lock. See docs/releasing.md, "Model lock". +# # Runs only on a schedule and by hand, on main: never on a pull request, so no fork's code ever # runs with these keys. on: @@ -158,3 +162,95 @@ jobs: run: | echo "::error::Model compatibility check failing. See the summary above and the issue titled \"$ISSUE_TITLE\"." exit 1 + + lock: + name: Refresh the model lock + needs: [harnesses, check] + if: always() && needs.harnesses.result == 'success' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # push the one refresh branch + pull-requests: write # open or update its pull request + env: + BRANCH: model-lock/refresh + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: model-compat-* + path: ${{ runner.temp }}/results + merge-multiple: true + - name: Write the lock from the source images' results + id: lock + run: | + set -euo pipefail + shopt -s nullglob + files=("$RUNNER_TEMP"/results/*.json) + if [ "${#files[@]}" -eq 0 ]; then + echo "No harness job wrote results; the lock stays as it is." + echo "changed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + bun scripts/model-compat.ts lock --changes "$RUNNER_TEMP/lock-changes.md" "${files[@]}" + if git diff --quiet -- shared/model-lock.json; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + { echo "## Model lock changes"; echo; cat "$RUNNER_TEMP/lock-changes.md"; } >> "$GITHUB_STEP_SUMMARY" + fi + # GITHUB_TOKEN's pull requests get CI runs that wait for a maintainer to approve them. A + # GitHub App installation token, when one is configured, has no such limit. + - id: app-token + if: steps.lock.outputs.changed == 'true' && vars.OPENBOT_UPGRADE_APP_CLIENT_ID != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.OPENBOT_UPGRADE_APP_CLIENT_ID }} + private-key: ${{ secrets.OPENBOT_UPGRADE_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + - name: Open or update the refresh pull request + if: steps.lock.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.GITHUB_TOKEN }} + TOKEN_KIND: ${{ steps.app-token.outputs.token && 'app' || 'github-token' }} + # Credentials for this one push, through gh, and no hook can run. + GIT_CONFIG_COUNT: "3" + GIT_CONFIG_KEY_0: credential.helper + GIT_CONFIG_VALUE_0: "" + GIT_CONFIG_KEY_1: credential.helper + GIT_CONFIG_VALUE_1: "!gh auth git-credential" + GIT_CONFIG_KEY_2: core.hooksPath + GIT_CONFIG_VALUE_2: /dev/null + run: | + set -euo pipefail + git switch -C "$BRANCH" + git -c user.name="github-actions[bot]" \ + -c user.email="41898282+github-actions[bot]@users.noreply.github.com" \ + commit -m "Refresh the model lock from the daily model check" -- shared/model-lock.json + git push --force "https://github.com/$GITHUB_REPOSITORY.git" "HEAD:refs/heads/$BRANCH" + body="$RUNNER_TEMP/lock-pr.md" + { + echo "The daily model check found models that started or stopped answering through the clients the harnesses ship. This refreshes \`shared/model-lock.json\`, the models the picker may offer for each harness." + echo + cat "$RUNNER_TEMP/lock-changes.md" + echo + echo "From $RUN_URL. Every run that finds a change updates this pull request. See docs/releasing.md, \"Model lock\"." + if [ "$TOKEN_KIND" != app ]; then + echo + echo "**CI must be started by hand.** This pull request was opened with \`GITHUB_TOKEN\`, so its CI runs wait for a maintainer to approve them." + fi + } > "$body" + number=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open --json number --jq ".[0].number // empty") + if [ -n "$number" ]; then + gh pr edit "$number" --repo "$GITHUB_REPOSITORY" --body-file "$body" + else + gh pr create --repo "$GITHUB_REPOSITORY" --base main --head "$BRANCH" \ + --title "Refresh the model lock" --body-file "$body" + fi diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml new file mode 100644 index 0000000000..0785135dee --- /dev/null +++ b/.github/workflows/model-lock.yml @@ -0,0 +1,103 @@ +name: Model lock + +# A pull request that changes a harness's model client (its requirements, its package.json or +# lockfile, its Dockerfile, or the ChatGPT plan's Codex client version) must not break a model the +# model lock (shared/model-lock.json) says that harness runs: a person set to it would get the +# provider's refusal on every message. This builds the changed harnesses from the pull request, +# sends each listed model one request through them (scripts/model-compat.ts), and fails when a +# model the lock verifies no longer answers. See docs/releasing.md, "Model lock". +# +# Only for pull requests from this repository: a fork's pull request is never given the provider +# keys, so it is skipped and the daily check covers it once it merges. +on: + pull_request: + paths: + - "agent-*/requirements.txt" + - "agent-*/Dockerfile" + - "agent-*/package.json" + - "agent-*/bun.lock" + - "agent-langgraph-agui/src/chatgpt_models.py" + - "shared/model-lock.json" + - "scripts/model-compat.ts" + - ".github/workflows/model-lock.yml" + +permissions: {} + +concurrency: + group: model-lock-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + changed: + name: Harnesses whose client changed + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + outputs: + harnesses: ${{ steps.list.outputs.harnesses }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + - id: list + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + # The check itself, or the lock, changed: every harness the lock names is checked. + if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then + harnesses=$(jq -c '[.engines | keys[]]' shared/model-lock.json) + else + harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \ + | jq -R . | jq -sc --slurpfile published .github/published-images.json \ + 'map(select(. as $h | $published[0] | index($h)))') + fi + echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT" + echo "Harnesses to check: $harnesses" + + check: + name: ${{ matrix.harness }} + needs: changed + if: needs.changed.outputs.harnesses != '[]' + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + harness: ${{ fromJSON(needs.changed.outputs.harnesses) }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 + - name: One request per model through this pull request's harness + env: + HARNESS: ${{ matrix.harness }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + run: >- + bun scripts/model-compat.ts --harness "$HARNESS" --tags source --exit-zero + --out "$RUNNER_TEMP/model-compat.json" + - name: Every model the lock verifies still answers + env: + HARNESS: ${{ matrix.harness }} + run: | + set +e + bun scripts/model-compat.ts guard --summary "$RUNNER_TEMP/guard.md" "$RUNNER_TEMP/model-compat.json" + code=$? + cat "$RUNNER_TEMP/guard.md" >> "$GITHUB_STEP_SUMMARY" + if [ "$code" -ne 0 ]; then + echo "::error::A model the lock verifies for $HARNESS no longer answers through this change's client. See the summary." + fi + exit "$code" diff --git a/api/openapi.yaml b/api/openapi.yaml index f76767b282..fde7de9524 100644 --- a/api/openapi.yaml +++ b/api/openapi.yaml @@ -23707,6 +23707,8 @@ components: type: - string - 'null' + unverified: + $ref: '#/components/schemas/UnverifiedModel' ModelChoiceInput: type: object additionalProperties: false @@ -24843,6 +24845,19 @@ components: text: type: string description: What was said. + UnverifiedModel: + type: object + description: The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + required: + - model + - label + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. AppSource: type: string enum: diff --git a/api/openapi/model-connections.yaml b/api/openapi/model-connections.yaml index e42d9df16a..590bb25e74 100644 --- a/api/openapi/model-connections.yaml +++ b/api/openapi/model-connections.yaml @@ -587,6 +587,23 @@ components: type: [string, 'null'] effort: type: [string, 'null'] + unverified: + $ref: '#/components/schemas/UnverifiedModel' + UnverifiedModel: + type: object + description: >- + The model a person chose for this conversation, set aside because it + hasn't been checked to work with the client this Bot's harness + ships. Runs answer on the default model until it has been. Absent when + the choice is in use, or set aside for another reason. + required: [model, label] + properties: + model: + type: string + description: The chosen model's id. + label: + type: string + description: Its name, as a model picker shows it. ModelChoiceInput: type: object additionalProperties: false diff --git a/app/src/lib/api/generated/model/index.ts b/app/src/lib/api/generated/model/index.ts index 814ac6b88d..b3e90e5219 100644 --- a/app/src/lib/api/generated/model/index.ts +++ b/app/src/lib/api/generated/model/index.ts @@ -1456,6 +1456,7 @@ export * from "./unpinBotHeaders"; export * from "./unpublishComponentHeaders"; export * from "./unpublishTeamBotHeaders"; export * from "./unsupportedFileResponse"; +export * from "./unverifiedModel"; export * from "./updateAppHeaders"; export * from "./updateAppRequest"; export * from "./updateApprovalPreferencesHeaders"; diff --git a/app/src/lib/api/generated/model/modelChoice.ts b/app/src/lib/api/generated/model/modelChoice.ts index 3b7cddda63..6d84c877ce 100644 --- a/app/src/lib/api/generated/model/modelChoice.ts +++ b/app/src/lib/api/generated/model/modelChoice.ts @@ -1,6 +1,7 @@ /** * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. */ +import type { UnverifiedModel } from "./unverifiedModel"; /** * The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default. @@ -10,4 +11,5 @@ export interface ModelChoice { model: string | null; /** @nullable */ effort: string | null; + unverified?: UnverifiedModel; } diff --git a/app/src/lib/api/generated/model/unverifiedModel.ts b/app/src/lib/api/generated/model/unverifiedModel.ts new file mode 100644 index 0000000000..c2cedf19df --- /dev/null +++ b/app/src/lib/api/generated/model/unverifiedModel.ts @@ -0,0 +1,13 @@ +/** + * Generated by Orval from api/openapi.yaml. Do not edit: change the spec and run `bun run api:generate`. + */ + +/** + * The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason. + */ +export interface UnverifiedModel { + /** The chosen model's id. */ + model: string; + /** Its name, as a model picker shows it. */ + label: string; +} diff --git a/app/src/lib/api/generated/models/models.zod.ts b/app/src/lib/api/generated/models/models.zod.ts index 296e895d13..ea9d04eb29 100644 --- a/app/src/lib/api/generated/models/models.zod.ts +++ b/app/src/lib/api/generated/models/models.zod.ts @@ -134,6 +134,15 @@ export const GetModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", @@ -216,6 +225,15 @@ export const ReplaceModelChoiceResponse = zod .object({ model: zod.string().nullable(), effort: zod.string().nullable(), + unverified: zod + .object({ + model: zod.string().describe("The chosen model's id."), + label: zod.string().describe("Its name, as a model picker shows it."), + }) + .optional() + .describe( + "The model a person chose for this conversation, set aside because it hasn't been checked to work with the client this Bot's harness ships. Runs answer on the default model until it has been. Absent when the choice is in use, or set aside for another reason.", + ), }) .describe( "The model and thinking effort a person chose for one conversation. Null model: the default model. Null effort: none is sent, so the model thinks on its own default.", diff --git a/app/src/lib/models/queries.ts b/app/src/lib/models/queries.ts index 9b126f7d76..f0a4612e24 100644 --- a/app/src/lib/models/queries.ts +++ b/app/src/lib/models/queries.ts @@ -80,6 +80,12 @@ export type AvailableModels = { export type ConversationModel = { model: string | null; effort: string | null; + /** + * The model the person chose here, set aside because it hasn't been checked to work with this + * Bot's harness (shared/model-lock.ts). Runs use the default until it has been. Only ever on a + * read from the server. + */ + unverified?: { model: string; label: string }; }; export const modelKeys = { diff --git a/app/src/lib/models/run-model.ts b/app/src/lib/models/run-model.ts index c90768375c..7937dda6bf 100644 --- a/app/src/lib/models/run-model.ts +++ b/app/src/lib/models/run-model.ts @@ -28,6 +28,15 @@ export function withdrawnModelNotice( return `${picked} ${gone} any more, so this conversation now uses ${now}. Pick another model to change it.`; } +/** + * What the composer says when the model a person chose here hasn't been proven to work with this + * Bot (shared/model-lock.ts): the provider would turn down every message, so runs use the default + * until a check proves it. + */ +export function unverifiedModelNotice(picked: string, now: string): string { + return `${picked} doesn't work with this Bot yet, so this conversation uses ${now}. Pick another model to change it.`; +} + /** * The model and thinking effort one conversation runs on, as its picker shows them, and the way * to change them. @@ -133,8 +142,19 @@ export function useConversationModel( ? undefined : shownPick(picked, catalogue), saving: save.isPending, - /** What the composer says about the pick, when the picked model stopped being offered. */ - notice: withdrawn?.channelId === channelId ? withdrawn.notice : null, + /** + * What the composer says about the pick: the chosen model doesn't work with this Bot yet, or + * it stopped being offered. + */ + notice: + (!save.isPending && saved.data?.unverified && catalogue + ? unverifiedModelNotice( + saved.data.unverified.label, + catalogue.models.find((option) => option.id === catalogue.default) + ?.label ?? catalogue.default, + ) + : null) ?? + (withdrawn?.channelId === channelId ? withdrawn.notice : null), // Read the ref at submission time, including a choice made before React renders again. beforeSend: async () => { await pendingSave.current; diff --git a/docs/releasing.md b/docs/releasing.md index 533be73faa..f305456962 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1125,6 +1125,37 @@ bun scripts/model-compat.ts --env-file .env --harness agent-claude-sdk --provide `--model ` narrows the models, `--tags latest` skips building from source. +### Model lock + +The picker offers a model only when this check has proven it on the Bot's harness. +`shared/model-lock.json` holds, per harness, the model clients its `source` image carried and, per +provider, the models that answered and the ones refused, in the provider's words. The server's model +catalogue reads it (`shared/model-lock.ts` has the rule): a live model the lock doesn't verify for +the harness is hidden, and a conversation already set to one runs on the default and says so. + +- **Which harness.** A Claude plan runs every Bot through `agent-claude-sdk`, a ChatGPT plan through + `agent-langgraph-agui`, and with a key, the harness picked in setup when it takes OpenBot's model + choice. OpenBot's own runtime is not a harness image, so its models are not locked. +- **What is locked.** Only a harness and provider pair this check has run. A pair it never ran (a + provider with no key here, such as xAI today) stays open, and a gateway is never locked. +- **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull + request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that + started and stopped answering and the clients that moved. Merging it lets the picker offer them. + A failure that says nothing about the model (the harness never started, a timeout) keeps what the + lock said before. +- **A client bump** is held to the lock: a pull request from this repository that changes a + harness's `requirements.txt`, `package.json`, `bun.lock`, `Dockerfile` or the Codex client version + runs the **Model lock** workflow, which builds that harness, sends every listed model a request, + and fails when a model the lock verifies no longer answers. Models the bump newly fixes are added + by the next daily refresh. + +To write the lock from results by hand: + +```sh +bun scripts/model-compat.ts lock --changes changes.md results/*.json +bun scripts/model-compat.ts guard results/*.json +``` + ## The one thing CI cannot do The smoke journey in `tests/smoke` is the only check that proves the parts are wired to each other: diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index fa4bd8ad47..d0e28ec44c 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -25,18 +25,30 @@ * OPENAI_API_KEY, GEMINI_API_KEY, XAI_API_KEY. A provider with no key is reported as skipped, * not failed. Keys are never printed: every detail is scrubbed of them. * + * What passed on the `source` image becomes the model lock (shared/model-lock.ts): the models + * the picker may offer for each harness. `lock` writes it from the results; `guard` fails when a + * model the lock verifies no longer answers, for a pull request that changes a harness's client. + * * bun scripts/model-compat.ts [--env-file .env] [--harness agent-claude-sdk,...] * [--provider anthropic,...] [--model ] [--tags latest,source] [--concurrency 6] * [--out results.json] [--exit-zero] * bun scripts/model-compat.ts report --summary [--issue ] results/*.json + * bun scripts/model-compat.ts lock [--lock shared/model-lock.json] [--date YYYY-MM-DD] results/*.json + * bun scripts/model-compat.ts guard [--lock shared/model-lock.json] [--summary ] results/*.json * * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of - * several runs, writes the Markdown summary, and exits 1 when any of them failed. + * several runs, writes the Markdown summary, and exits 1 when any of them failed. `lock` exits 0, + * and says whether the lock changed. `guard` exits 1 when a verified model failed. */ import { randomBytes } from "node:crypto"; -import { readFileSync, writeFileSync } from "node:fs"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import type { + LockedEngine, + LockProvider, + ModelLock, +} from "../shared/model-lock"; const ROOT = join(import.meta.dir, ".."); const REGISTRY = "ghcr.io/copilotkit/openbot-"; @@ -304,7 +316,8 @@ export const PROVIDERS: readonly Provider[] = [ /** * Providers the Desktop offers by address that this check leaves out on purpose, with why. A - * gateway serves its own aliases, not a provider's models. + * gateway serves its own aliases, not a provider's models, so the lock has nothing to say about + * it and leaves its models open. */ export const NOT_CHECKED: readonly { baseUrl: string; why: string }[] = [ { @@ -540,12 +553,22 @@ export function codexGate( export type Result = { harness: string; tag: string; + /** What a person reads: "Anthropic (key or Claude plan)", "ChatGPT plan". */ provider: string; + /** What the lock files it under. */ + providerId: LockProvider; model: string; status: "pass" | "fail" | "skipped"; detail: string; }; +/** One results file: the rows, and the model clients each image carried. */ +export type Results = { + results: Result[]; + /** By harness, then image tag: package name to version. */ + clients?: Record>>; +}; + function cell(text: string): string { return oneLine(text).replace(/\|/g, "\\|"); } @@ -768,6 +791,100 @@ async function prepare(harness: Harness, tag: string) { : exec(["docker", "pull", "--quiet", imageOf(harness, tag)]); } +/** + * The model-client packages whose versions decide what a provider accepts, by the names each + * ecosystem installs them under. Whichever an image has are recorded in the lock beside the + * models they were proven with, so a lock entry says what it was true for. + */ +const PYTHON_CLIENTS = [ + "claude-agent-sdk", + "ag-ui-claude-sdk", + "openai", + "anthropic", + "litellm", + "google-genai", + "google-adk", + "ag2", + "agno", + "crewai", + "langgraph", + "langchain-core", + "langchain-openai", + "langchain-anthropic", + "langchain-google-genai", + "langroid", + "llama-index-core", + "llama-index-llms-openai", + "llama-index-llms-anthropic", + "llama-index-llms-google-genai", + "pydantic-ai-slim", + "strands-agents", + "agent-framework", + "agent-framework-core", +]; +const NODE_CLIENTS = [ + "@mastra/core", + "ai", + "@ai-sdk/openai", + "@ai-sdk/anthropic", + "@ai-sdk/google", +]; + +const PYTHON_VERSIONS = ` +import json, importlib.metadata as m +found = {} +for name in ${JSON.stringify(PYTHON_CLIENTS)}: + try: + found[name] = m.version(name) + except Exception: + pass +try: + import claude_agent_sdk, pathlib, subprocess + for cli in pathlib.Path(claude_agent_sdk.__file__).parent.glob("_bundled/claude*"): + said = subprocess.run([str(cli), "--version"], capture_output=True, text=True, timeout=30).stdout.split() + if said: + found["claude-code"] = said[0] + break +except Exception: + pass +print(json.dumps(found)) +`; +const NODE_VERSIONS = ` +const found = {}; +for (const name of ${JSON.stringify(NODE_CLIENTS)}) { + try { found[name] = require(\`\${process.cwd()}/node_modules/\${name}/package.json\`).version; } catch {} +} +console.log(JSON.stringify(found)); +`; + +/** The model clients an image carries, or an empty record when it says nothing. */ +async function clientsIn(image: string): Promise> { + for (const args of [ + ["python", "-c", PYTHON_VERSIONS], + ["bun", "-e", NODE_VERSIONS], + ] as const) { + const ran = await exec([ + "docker", + "run", + "--rm", + "--entrypoint", + args[0], + image, + args[1], + args[2], + ]); + if (ran.code !== 0) continue; + try { + const found = JSON.parse(ran.stdout.trim().split("\n").at(-1) ?? ""); + if (found && typeof found === "object" && Object.keys(found).length) + return found as Record; + } catch { + // Not this runtime's answer: try the next. + } + } + return {}; +} + let serial = 0; /** Starts the harness on one provider and model, sends one run, and stops it. */ @@ -987,6 +1104,7 @@ async function check(argv: string[]): Promise { harness: row.harness.image ?? "", tag: options.tags.join(", "), provider: label(row.provider), + providerId: row.provider, model: "(all)", status: "skipped", detail: `skipped: no key (${provider(row.provider).secret})`, @@ -996,6 +1114,7 @@ async function check(argv: string[]): Promise { harness: row.harness.image ?? "", tag: options.tags.join(", "), provider: label(row.provider), + providerId: row.provider, model: "(all)", status: "fail", detail: row.detail, @@ -1020,6 +1139,15 @@ async function check(argv: string[]): Promise { ); }); + // The model clients each image carries, which the lock records beside what passed. + const clients: NonNullable = {}; + for (const { harness, tag } of images) { + const image = imageOf(harness, tag); + if (unpulled.has(image) || !harness.image) continue; + const found = await clientsIn(image); + clients[harness.image] = { ...clients[harness.image], [tag]: found }; + } + // The ChatGPT plan's Codex client version, in whichever images carry the plan. const codex = await fetchJson(CODEX_CATALOGUE); if (!options.providers || options.providers.includes("openai")) @@ -1044,6 +1172,7 @@ async function check(argv: string[]): Promise { harness: harness.image ?? "", tag, provider: "ChatGPT plan", + providerId: "chatgpt", model: "(all)", status: "fail", detail: `could not read Codex's model catalogue: ${codex.error}`, @@ -1055,6 +1184,7 @@ async function check(argv: string[]): Promise { harness: harness.image ?? "", tag, provider: "ChatGPT plan", + providerId: "chatgpt", ...gate, }); } @@ -1078,6 +1208,7 @@ async function check(argv: string[]): Promise { row.provider === "anthropic" && row.harness.credential === "anthropic" ? "Anthropic (key or Claude plan)" : label(row.provider), + providerId: row.provider, model: row.model, ...outcome, }); @@ -1088,13 +1219,268 @@ async function check(argv: string[]): Promise { }); if (options.out) - writeFileSync(options.out, `${JSON.stringify({ results }, null, 2)}\n`); + writeFileSync( + options.out, + `${JSON.stringify({ results, clients } satisfies Results, null, 2)}\n`, + ); console.log(renderSummary(results)); return results.some((row) => row.status === "fail") && !options.exitZero ? 1 : 0; } +// --------------------------------------------------------------------------------------------- +// The model lock. +// --------------------------------------------------------------------------------------------- + +const PROVIDER_IDS_BY_LABEL: Record = { + OpenAI: "openai", + Anthropic: "anthropic", + "Anthropic (key or Claude plan)": "anthropic", + Google: "google", + xAI: "xai", + "ChatGPT plan": "chatgpt", +}; + +/** The lock's name for a row's provider. Results written before rows carried it are read by label. */ +export function providerIdOf(row: { + provider: string; + providerId?: string; +}): LockProvider | undefined { + if (row.providerId) return row.providerId as LockProvider; + return PROVIDER_IDS_BY_LABEL[row.provider]; +} + +/** + * A failure that says nothing about the model: the harness never started, the image could not be + * had, or the run timed out with no word from the provider. The lock keeps what it knew before + * for such a model rather than take it away on a bad day for the runner. + */ +const INCONCLUSIVE = + /^(no answer:|the harness did not start|the harness never became healthy|the harness published no port|could not get |the stream ended without finishing the run)/; + +export function inconclusive(detail: string): boolean { + return INCONCLUSIVE.test(detail); +} + +function sortedRecord(record: Record): Record { + return Object.fromEntries( + Object.entries(record).sort(([a], [b]) => a.localeCompare(b)), + ); +} + +/** + * The tag a harness's lock is read from: `source`, the image built from this commit, which is what + * the next release ships. A run of the published images alone says nothing about this commit. + */ +function lockTag(rows: readonly Result[]): string | undefined { + return rows.some((row) => row.tag === SOURCE) ? SOURCE : undefined; +} + +/** + * The model lock from a run's results, over the lock before it. + * + * For each harness, the `source` image's rows: what passed is verified, what the provider refused + * is refused, in its words. What the run could not decide (an inconclusive failure), a provider + * it had no key for or could not list, and a harness it did not run, keep what the lock said + * before. A model the provider no longer lists drops out. The date moves only when the lock does, + * so a day with nothing new changes nothing. + */ +export function buildLock( + previous: ModelLock | undefined, + runs: readonly Results[], + date: string, +): ModelLock { + const results = runs.flatMap((run) => run.results); + const clients = Object.assign({}, ...runs.map((run) => run.clients ?? {})); + const engines: Record = { + ...(previous?.engines ?? {}), + }; + const harnesses = [...new Set(results.map((row) => row.harness))]; + for (const harness of harnesses) { + const rows = results.filter((row) => row.harness === harness); + const tag = lockTag(rows); + const before = previous?.engines[harness]; + const providers: LockedEngine["providers"] = { + ...(before?.providers ?? {}), + }; + const ran = rows.filter((row) => row.tag === tag && row.model !== "(all)"); + for (const id of new Set(ran.map(providerIdOf))) { + if (!id) continue; + const earlier = before?.providers[id]; + const verified = new Set(); + const refused: Record = {}; + for (const row of ran.filter((row) => providerIdOf(row) === id)) { + if (row.status === "pass") verified.add(row.model); + else if (row.status !== "fail") continue; + else if (!inconclusive(row.detail)) refused[row.model] = row.detail; + else if (earlier?.verified.includes(row.model)) verified.add(row.model); + else if (earlier?.refused[row.model] !== undefined) + refused[row.model] = earlier.refused[row.model] ?? ""; + } + providers[id] = { + verified: [...verified].sort(), + refused: sortedRecord(refused), + }; + } + if (!Object.keys(providers).length) continue; + const found = tag ? clients[harness]?.[tag] : undefined; + engines[harness] = { + clients: sortedRecord( + found && Object.keys(found).length ? found : (before?.clients ?? {}), + ), + providers: sortedRecord(providers), + }; + } + const next = sortedRecord(engines); + const same = + previous !== undefined && + JSON.stringify(previous.engines) === JSON.stringify(next); + return { checkedAt: same ? previous.checkedAt : date, engines: next }; +} + +/** + * What changed between two locks, as a pull request says it: models that newly answer, models + * that stopped, models that dropped out of a provider's listing, and clients that moved. + */ +export function describeLockChange( + before: ModelLock | undefined, + after: ModelLock, +): string { + const lines: string[] = []; + for (const [engine, entry] of Object.entries(after.engines)) { + const earlier = before?.engines[engine]; + const changes: string[] = []; + for (const [provider, now] of Object.entries(entry.providers)) { + if (!now) continue; + const was = earlier?.providers[provider as LockProvider]; + const known = new Set([ + ...(was?.verified ?? []), + ...Object.keys(was?.refused ?? {}), + ]); + const passes = now.verified.filter( + (model) => !was?.verified.includes(model), + ); + const stopped = Object.keys(now.refused).filter((model) => + was?.verified.includes(model), + ); + const newlyRefused = Object.keys(now.refused).filter( + (model) => !known.has(model), + ); + const gone = [...known].filter( + (model) => !now.verified.includes(model) && !(model in now.refused), + ); + if (passes.length) + changes.push( + `${provider}: now answers ${passes.map((m) => `\`${m}\``).join(", ")}`, + ); + if (stopped.length) + changes.push( + `${provider}: **stopped answering** ${stopped.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (newlyRefused.length) + changes.push( + `${provider}: refused ${newlyRefused.map((m) => `\`${m}\` (${cell(now.refused[m] ?? "")})`).join(", ")}`, + ); + if (gone.length) + changes.push( + `${provider}: no longer listed ${gone.map((m) => `\`${m}\``).join(", ")}`, + ); + } + const moved = Object.entries(entry.clients) + .filter(([name, version]) => earlier?.clients[name] !== version) + .map(([name, version]) => + earlier?.clients[name] + ? `${name} ${earlier.clients[name]} to ${version}` + : `${name} ${version}`, + ); + if (moved.length) changes.push(`clients: ${moved.join(", ")}`); + if (changes.length) + lines.push( + `- **${engine}**`, + ...changes.map((change) => ` - ${change}`), + ); + } + return lines.length ? lines.join("\n") : "No model changed."; +} + +/** Every model the lock verifies that now fails: what a client bump must not break. */ +export function brokenVerified( + lock: ModelLock, + results: readonly Result[], +): Result[] { + return results.filter((row) => { + if (row.status !== "fail" || row.model === "(all)") return false; + const id = providerIdOf(row); + return ( + !!id && + (lock.engines[row.harness]?.providers[id]?.verified.includes(row.model) ?? + false) + ); + }); +} + +function readResults(files: readonly string[]): Results[] { + return files.map((file) => JSON.parse(readFileSync(file, "utf8")) as Results); +} + +function lockCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let date = new Date().toISOString().slice(0, 10); + let changes: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--date") date = argv[++i] ?? date; + else if (arg === "--changes") changes = argv[++i]; + else if (arg) files.push(arg); + } + const previous = existsSync(path) + ? (JSON.parse(readFileSync(path, "utf8")) as ModelLock) + : undefined; + const next = buildLock(previous, readResults(files), date); + const text = `${JSON.stringify(next, null, 2)}\n`; + const changed = !existsSync(path) || readFileSync(path, "utf8") !== text; + if (changed) writeFileSync(path, text); + if (changes) + writeFileSync(changes, `${describeLockChange(previous, next)}\n`); + console.log( + changed + ? `The model lock changed: ${path}` + : "The model lock is unchanged.", + ); + return 0; +} + +function guardCommand(argv: string[]): number { + let path = join(ROOT, "shared/model-lock.json"); + let summary: string | undefined; + const files: string[] = []; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === "--lock") path = argv[++i] ?? path; + else if (arg === "--summary") summary = argv[++i]; + else if (arg) files.push(arg); + } + const lock = JSON.parse(readFileSync(path, "utf8")) as ModelLock; + const results = readResults(files).flatMap((run) => run.results); + const broken = brokenVerified(lock, results); + const markdown = [ + "## Model lock", + "", + broken.length + ? `${broken.length} model${broken.length === 1 ? "" : "s"} the lock verifies no longer answer${broken.length === 1 ? "s" : ""} through this change's client. A person set to one would get the provider's refusal on every message. Fix the client, or, if the model really can't run on this harness any more, say so in the pull request: the next daily check takes it out of the lock.` + : "Every model the lock verifies for the harnesses this change touches still answers.", + "", + ...(broken.length ? [renderTable(broken), ""] : []), + renderSummary(results), + ].join("\n"); + if (summary) writeFileSync(summary, `${markdown}\n`); + else console.log(markdown); + return broken.length ? 1 : 0; +} + function report(argv: string[]): number { let summary: string | undefined; let issue: string | undefined; @@ -1128,6 +1514,13 @@ function report(argv: string[]): number { if (import.meta.main) { const argv = process.argv.slice(2); - const code = argv[0] === "report" ? report(argv.slice(1)) : await check(argv); + const code = + argv[0] === "report" + ? report(argv.slice(1)) + : argv[0] === "lock" + ? lockCommand(argv.slice(1)) + : argv[0] === "guard" + ? guardCommand(argv.slice(1)) + : await check(argv); process.exit(code); } diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts new file mode 100644 index 0000000000..705048f544 --- /dev/null +++ b/scripts/model-lock.test.ts @@ -0,0 +1,232 @@ +import { expect, test } from "bun:test"; +import type { ModelLock } from "../shared/model-lock"; +import { + brokenVerified, + buildLock, + describeLockChange, + providerIdOf, + type Result, + type Results, +} from "./model-compat"; + +/* + * Writing the model lock from the model check's results, and holding a client bump to it. The + * rows are what the check reported on 2026-11-17, in its own words. + */ + +const OPUS_REFUSED = + "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again."; + +const row = ( + model: string, + status: Result["status"], + detail = "OK", + extra: Partial = {}, +): Result => ({ + harness: "agent-claude-sdk", + tag: "source", + provider: "Anthropic (key or Claude plan)", + providerId: "anthropic", + model, + status, + detail, + ...extra, +}); + +const run: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + // `latest` is what people run today; the lock is what the next release ships. + row("claude-sonnet-5-5", "fail", OPUS_REFUSED, { tag: "latest" }), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + latest: { "claude-agent-sdk": "0.2.150" }, + }, + }, +}; + +test("what answered on the source image is verified, what was refused is kept in the provider's words", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + expect(lock).toEqual({ + checkedAt: "2026-11-17", + engines: { + "agent-claude-sdk": { + clients: { "claude-agent-sdk": "0.2.152", "claude-code": "2.1.259" }, + providers: { + anthropic: { + verified: ["claude-haiku-5-5", "claude-sonnet-5-5"], + refused: { "claude-opus-5-5": OPUS_REFUSED }, + }, + }, + }, + }, + }); +}); + +test("a bad day for the runner takes nothing away", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const flaky: Results = { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row( + "claude-haiku-5-5", + "fail", + "the harness never became healthy: exit 1", + ), + row("claude-opus-5-5", "fail", "no answer: TimeoutError"), + ], + }; + const after = buildLock(before, [flaky], "2026-11-18"); + expect(after).toEqual(before); + // Nothing changed, so neither did the date, and the daily check opens no pull request. + expect(after.checkedAt).toBe("2026-11-17"); +}); + +test("a model that starts answering after a client bump is verified, and the clients move with it", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const bumped: Results = { + results: [ + row("claude-sonnet-5-5", "pass"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + row("claude-fable-5-2", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }; + const after = buildLock(before, [bumped], "2026-11-18"); + expect(after.checkedAt).toBe("2026-11-18"); + expect(after.engines["agent-claude-sdk"]).toEqual({ + clients: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + providers: { + anthropic: { + verified: [ + "claude-fable-5-2", + "claude-haiku-5-5", + "claude-opus-5-5", + "claude-sonnet-5-5", + ], + refused: {}, + }, + }, + }); +}); + +test("a provider with no key and a harness that did not run keep what was known", () => { + const before: ModelLock = { + checkedAt: "2026-11-10", + engines: { + "agent-agno": { + clients: { agno: "2.0.0" }, + providers: { + xai: { verified: ["grok-4.7"], refused: {} }, + openai: { verified: ["gpt-6.1-terra"], refused: {} }, + }, + }, + "agent-adk": { + clients: { "google-adk": "1.0.0" }, + providers: { openai: { verified: ["gpt-5.5"], refused: {} } }, + }, + }, + }; + const today: Results = { + results: [ + { + harness: "agent-agno", + tag: "latest, source", + provider: "xAI", + providerId: "xai", + model: "(all)", + status: "skipped", + detail: "skipped: no key (XAI_API_KEY)", + }, + { + harness: "agent-agno", + tag: "source", + provider: "OpenAI", + providerId: "openai", + model: "gpt-6.1-terra", + status: "pass", + detail: "OK", + }, + ], + }; + const after = buildLock(before, [today], "2026-11-17"); + expect(after.engines["agent-agno"]?.providers.xai).toEqual({ + verified: ["grok-4.7"], + refused: {}, + }); + expect(after.engines["agent-adk"]).toEqual(before.engines["agent-adk"]!); + // No clients in the results: the ones recorded before stand. + expect(after.engines["agent-agno"]?.clients).toEqual({ agno: "2.0.0" }); +}); + +test("the refresh pull request says which models started and stopped answering", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const after = buildLock( + before, + [ + { + results: [ + row( + "claude-sonnet-5-5", + "fail", + "API Error: 400 invalid_request_error", + ), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "pass"), + ], + clients: { + "agent-claude-sdk": { + source: { "claude-agent-sdk": "0.2.170", "claude-code": "2.1.291" }, + }, + }, + }, + ], + "2026-11-18", + ); + const said = describeLockChange(before, after); + expect(said).toContain("now answers `claude-opus-5-5`"); + expect(said).toContain("**stopped answering** `claude-sonnet-5-5`"); + expect(said).toContain("claude-code 2.1.259 to 2.1.291"); + expect(describeLockChange(before, before)).toBe("No model changed."); +}); + +test("a run of the published images alone does not move the lock", () => { + const before = buildLock(undefined, [run], "2026-11-17"); + const latestOnly: Results = { + results: [row("claude-opus-5-5", "pass", "OK", { tag: "latest" })], + }; + expect(buildLock(before, [latestOnly], "2026-11-18")).toEqual(before); +}); + +test("results written before rows named their provider are read by its label", () => { + expect(providerIdOf({ provider: "Anthropic (key or Claude plan)" })).toBe( + "anthropic", + ); + expect(providerIdOf({ provider: "ChatGPT plan" })).toBe("chatgpt"); + expect(providerIdOf({ provider: "Google" })).toBe("google"); +}); + +test("a client bump that breaks a verified model is caught, and one the lock already refuses is not", () => { + const lock = buildLock(undefined, [run], "2026-11-17"); + const afterBump: Result[] = [ + row("claude-sonnet-5-5", "fail", "API Error: 400 invalid_request_error"), + row("claude-haiku-5-5", "pass"), + row("claude-opus-5-5", "fail", OPUS_REFUSED), + ]; + expect(brokenVerified(lock, afterBump).map((broken) => broken.model)).toEqual( + ["claude-sonnet-5-5"], + ); +}); diff --git a/shared/fixtures/model-listings/anthropic-models.json b/shared/fixtures/model-listings/anthropic-models.json new file mode 100644 index 0000000000..478a3cc111 --- /dev/null +++ b/shared/fixtures/model-listings/anthropic-models.json @@ -0,0 +1,1327 @@ +{ + "data": [ + { + "type": "model", + "id": "claude-fable-5-2", + "display_name": "Claude Fable 5.2", + "created_at": "2026-11-09T18:30:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-5-5", + "display_name": "Claude Haiku 5.5", + "created_at": "2026-10-07T18:00:00Z", + "line": "haiku", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5-5", + "display_name": "Claude Sonnet 5.5", + "created_at": "2026-09-28T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": true + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5-5", + "display_name": "Claude Opus 5.5", + "created_at": "2026-09-21T16:24:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5-1", + "display_name": "Claude Fable 5.1", + "created_at": "2026-08-28T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-5", + "display_name": "Claude Opus 5", + "created_at": "2026-07-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-5", + "display_name": "Claude Sonnet 5", + "created_at": "2026-06-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-fable-5", + "display_name": "Claude Fable 5", + "created_at": "2026-06-07T00:00:00Z", + "line": "fable", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": false + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-8", + "display_name": "Claude Opus 4.8", + "created_at": "2026-05-28T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-7", + "display_name": "Claude Opus 4.7", + "created_at": "2026-04-14T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": true + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": false + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-6", + "display_name": "Claude Sonnet 4.6", + "created_at": "2026-02-17T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-6", + "display_name": "Claude Opus 4.6", + "created_at": "2026-02-04T00:00:00Z", + "line": "opus", + "max_input_tokens": 1000000, + "max_tokens": 128000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": true + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": true + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": true + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-opus-4-5-20251101", + "display_name": "Claude Opus 4.5", + "created_at": "2025-11-24T00:00:00Z", + "line": "opus", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": true, + "low": { + "supported": true + }, + "medium": { + "supported": true + }, + "high": { + "supported": true + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-haiku-4-5-20251001", + "display_name": "Claude Haiku 4.5", + "created_at": "2025-10-15T00:00:00Z", + "line": "haiku", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": false + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "active", + "deprecated_at": null, + "retires_at": null + }, + { + "type": "model", + "id": "claude-sonnet-4-5-20250929", + "display_name": "Claude Sonnet 4.5", + "created_at": "2025-09-29T00:00:00Z", + "line": "sonnet", + "max_input_tokens": 200000, + "max_tokens": 64000, + "capabilities": { + "batch": { + "supported": true + }, + "citations": { + "supported": true + }, + "code_execution": { + "supported": true + }, + "context_management": { + "supported": true, + "clear_tool_uses_20250919": { + "supported": true + }, + "clear_thinking_20251015": { + "supported": true + }, + "compact_20260112": { + "supported": false + } + }, + "effort": { + "supported": false, + "low": { + "supported": false + }, + "medium": { + "supported": false + }, + "high": { + "supported": false + }, + "xhigh": { + "supported": false + }, + "max": { + "supported": false + } + }, + "image_input": { + "supported": true + }, + "pdf_input": { + "supported": true + }, + "server_tools": { + "supported": true, + "code_execution": { + "supported": true + }, + "web_search": { + "supported": true + } + }, + "structured_outputs": { + "supported": true + }, + "thinking": { + "supported": true, + "types": { + "enabled": { + "supported": true + }, + "adaptive": { + "supported": false + }, + "disabled": { + "supported": true + }, + "between_tools": { + "supported": false + } + } + } + }, + "lifecycle": "deprecated", + "deprecated_at": "2026-09-30T18:30:00Z", + "retires_at": "2026-11-30T17:00:00Z" + } + ], + "has_more": false, + "first_id": "claude-fable-5-2", + "last_id": "claude-sonnet-4-5-20250929" +} diff --git a/shared/fixtures/model-listings/openai-models.json b/shared/fixtures/model-listings/openai-models.json new file mode 100644 index 0000000000..f4c33cd814 --- /dev/null +++ b/shared/fixtures/model-listings/openai-models.json @@ -0,0 +1,705 @@ +{ + "object": "list", + "data": [ + { + "id": "gpt-5-nano-2025-08-07", + "object": "model", + "created": 1754426303, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-tts-2025-03-20", + "object": "model", + "created": 1765610731, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio-mini", + "object": "model", + "created": 1759512027, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe", + "object": "model", + "created": 1742068463, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.2", + "object": "model", + "created": 1765313051, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-2024-07-18", + "object": "model", + "created": 1721172717, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-2025-04-14", + "object": "model", + "created": 1744315746, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4.1-mini-2025-04-14", + "object": "model", + "created": 1744317547, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.4-nano-2026-03-17", + "object": "model", + "created": 1773450837, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-image-2.5-flare", + "object": "model", + "created": 1788563147, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-sol", + "object": "model", + "created": 1790552874, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5", + "object": "model", + "created": 1754425777, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o", + "object": "model", + "created": 1715367049, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-4o-mini-transcribe", + "object": "model", + "created": 1742068596, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-image-2-2026-04-21", + "object": "model", + "created": 1776399994, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-4o-mini-tts-2025-12-15", + "object": "model", + "created": 1765610837, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-audio", + "object": "model", + "created": 1756339249, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-nano", + "object": "model", + "created": 1773450870, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5.3-codex", + "object": "model", + "created": 1770537915, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api-2025-10-14", + "object": "model", + "created": 1760043960, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-mini-2025-12-15", + "object": "model", + "created": 1765760008, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-transcribe", + "object": "model", + "created": 1785168027, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1", + "object": "model", + "created": 1681940951, + "owned_by": "openai-internal", + "shutdown_date": "2027-01-06" + }, + { + "id": "omni-moderation-2024-09-26", + "object": "model", + "created": 1732734466, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2.1-mini", + "object": "model", + "created": 1782254706, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2025-08-28", + "object": "model", + "created": 1756271773, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-live-transcribe", + "object": "model", + "created": 1785168034, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini", + "object": "model", + "created": 1754425928, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.4-2026-03-05", + "object": "model", + "created": 1772654062, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-mini-2025-08-07", + "object": "model", + "created": 1754425867, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5-nano", + "object": "model", + "created": 1754426384, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-5.6-luna", + "object": "model", + "created": 1782228658, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-sunburst-2026-09-08", + "object": "model", + "created": 1788851010, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-flare", + "object": "model", + "created": 1793222647, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-2025-08-07", + "object": "model", + "created": 1754075360, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4o-mini-transcribe-2025-12-15", + "object": "model", + "created": 1765610407, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "o3-pro-2025-06-10", + "object": "model", + "created": 1749166761, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "chat-latest", + "object": "model", + "created": 1777704602, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro", + "object": "model", + "created": 1776894349, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-translate", + "object": "model", + "created": 1777950216, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-2025-08-28", + "object": "model", + "created": 1756256146, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-mini-transcribe-2025-03-20", + "object": "model", + "created": 1765610545, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.4-pro-2026-03-05", + "object": "model", + "created": 1772659657, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5-pro-2025-10-06", + "object": "model", + "created": 1759469707, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "tts-1-1106", + "object": "model", + "created": 1699053241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-pro-2026-04-23", + "object": "model", + "created": 1776894470, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro-2025-12-11", + "object": "model", + "created": 1765343959, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-sol", + "object": "model", + "created": 1782228018, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-astra", + "object": "model", + "created": 1793430042, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-2025-04-16", + "object": "model", + "created": 1744133301, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1-mini", + "object": "model", + "created": 1744318173, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-5.2-2025-12-11", + "object": "model", + "created": 1765313028, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "chatgpt-image-latest", + "object": "model", + "created": 1765925279, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-4o-mini", + "object": "model", + "created": 1721172741, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-image-2.5-sunburst", + "object": "model", + "created": 1788563162, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.5-flare-2026-09-08", + "object": "model", + "created": 1788851006, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-luna", + "object": "model", + "created": 1789406102, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "tts-1-hd-1106", + "object": "model", + "created": 1699053533, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-mini", + "object": "model", + "created": 1759517133, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-4o-transcribe-diarize", + "object": "model", + "created": 1750798887, + "owned_by": "system", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5.4", + "object": "model", + "created": 1772691852, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-audio-1.5", + "object": "model", + "created": 1771550885, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-1-mini", + "object": "model", + "created": 1758845821, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-image-1.5", + "object": "model", + "created": 1764030620, + "owned_by": "system", + "shutdown_date": "2026-12-01" + }, + { + "id": "gpt-5.1-2025-11-13", + "object": "model", + "created": 1762800353, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "text-embedding-3-small", + "object": "model", + "created": 1705948997, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.6-terra", + "object": "model", + "created": 1782228459, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-astra", + "object": "model", + "created": 1787853604, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.1-terra", + "object": "model", + "created": 1791908780, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6.2-sol", + "object": "model", + "created": 1793642820, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "omni-moderation-latest", + "object": "model", + "created": 1731689265, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini", + "object": "model", + "created": 1773451123, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-11-20", + "object": "model", + "created": 1739331543, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "tts-1-hd", + "object": "model", + "created": 1699046015, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-1.5", + "object": "model", + "created": 1771461469, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime-2", + "object": "model", + "created": 1778006032, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-realtime", + "object": "model", + "created": 1756271701, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "gpt-5.1", + "object": "model", + "created": 1762800673, + "owned_by": "system", + "shutdown_date": "2027-04-01" + }, + { + "id": "gpt-5-search-api", + "object": "model", + "created": 1759514629, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5-2026-04-23", + "object": "model", + "created": 1776839241, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "whisper-1", + "object": "model", + "created": 1677532384, + "owned_by": "openai-internal", + "shutdown_date": "2027-02-26" + }, + { + "id": "gpt-5-pro", + "object": "model", + "created": 1759469822, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-realtime-whisper", + "object": "model", + "created": 1778012060, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.2-pro", + "object": "model", + "created": 1765343983, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-live-1", + "object": "model", + "created": 1788889407, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-image-2.6-sunburst", + "object": "model", + "created": 1793222670, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.4-mini-2026-03-17", + "object": "model", + "created": 1773451076, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3-pro", + "object": "model", + "created": 1748475349, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-image-2", + "object": "model", + "created": 1776399795, + "owned_by": "system", + "shutdown_date": "2027-02-10" + }, + { + "id": "gpt-5.4-pro", + "object": "model", + "created": 1772659601, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-4o-mini-tts", + "object": "model", + "created": 1742403959, + "owned_by": "system", + "shutdown_date": "2027-01-06" + }, + { + "id": "gpt-realtime-mini-2025-12-15", + "object": "model", + "created": 1765612007, + "owned_by": "system", + "shutdown_date": "2027-01-20" + }, + { + "id": "text-embedding-ada-002", + "object": "model", + "created": 1671217299, + "owned_by": "openai-internal", + "shutdown_date": null + }, + { + "id": "gpt-4o-2024-08-06", + "object": "model", + "created": 1722814719, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "gpt-realtime-2.1", + "object": "model", + "created": 1782254687, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "o3", + "object": "model", + "created": 1744225308, + "owned_by": "system", + "shutdown_date": "2026-12-11" + }, + { + "id": "gpt-4.1", + "object": "model", + "created": 1744316542, + "owned_by": "system", + "shutdown_date": "2027-01-14" + }, + { + "id": "text-embedding-3-large", + "object": "model", + "created": 1705953180, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-5.5", + "object": "model", + "created": 1776824847, + "owned_by": "system", + "shutdown_date": null + }, + { + "id": "gpt-6-sol", + "object": "model", + "created": 1789405937, + "owned_by": "system", + "shutdown_date": null + } + ] +} diff --git a/shared/model-lock.json b/shared/model-lock.json new file mode 100644 index 0000000000..6200ff35f2 --- /dev/null +++ b/shared/model-lock.json @@ -0,0 +1,4 @@ +{ + "checkedAt": "2026-11-17", + "engines": {} +} diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts new file mode 100644 index 0000000000..c9b638241d --- /dev/null +++ b/shared/model-lock.test.ts @@ -0,0 +1,90 @@ +import { expect, test } from "bun:test"; +import anthropicListing from "./fixtures/model-listings/anthropic-models.json"; +import openaiListing from "./fixtures/model-listings/openai-models.json"; +import { lockAllows, lockVerdict, MODEL_LOCK } from "./model-lock"; + +/* + * The lock's rule against real data: the model listings OpenAI and Anthropic answered on + * 2026-11-17 (shared/fixtures/model-listings), and the lock the model check wrote that day. The + * Claude plan's failure that day was the Claude Agent SDK harness offering Opus 5.5 and Fable + * 5.2, which its pinned Claude Code was refused on every message. + */ + +const anthropicIds = anthropicListing.data.map((model) => model.id); +const openaiIds = openaiListing.data.map((model) => model.id); + +const offered = ( + engine: string | undefined, + provider: "anthropic" | "openai", + ids: string[], +) => ids.filter((id) => lockAllows(MODEL_LOCK, engine, provider, id)); + +test("the Claude Agent SDK harness is not offered the models its Claude Code is refused", () => { + const claude = offered("agent-claude-sdk", "anthropic", anthropicIds); + expect(anthropicIds).toContain("claude-opus-5-5"); + expect(anthropicIds).toContain("claude-fable-5-2"); + expect(claude).not.toContain("claude-opus-5-5"); + expect(claude).not.toContain("claude-fable-5-2"); + expect(claude).toContain("claude-sonnet-5-5"); + expect(claude).toContain("claude-haiku-5-5"); + // Everything else Anthropic listed that day answered. + expect(claude).toHaveLength(anthropicIds.length - 2); +}); + +test("a refused model carries the provider's own words", () => { + const verdict = lockVerdict( + MODEL_LOCK, + "agent-claude-sdk", + "anthropic", + "claude-opus-5-5", + ); + expect(verdict.status).toBe("refused"); + expect(verdict.status === "refused" && verdict.said).toMatch( + /does not support this model; version [\d.]+ or newer is required/, + ); +}); + +test("a harness whose client takes every model is offered every model", () => { + expect(offered("agent-langgraph-agui", "anthropic", anthropicIds)).toEqual( + anthropicIds, + ); +}); + +test("the Responses-only OpenAI models are not offered on a Chat Completions harness", () => { + const ag2 = offered("agent-ag2", "openai", openaiIds); + for (const model of ["gpt-5-pro", "o3-pro"]) + if (openaiIds.includes(model)) expect(ag2).not.toContain(model); + expect(ag2.length).toBeGreaterThan(0); +}); + +test("a model released after the last check waits for it", () => { + expect( + lockVerdict(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toEqual({ status: "unchecked" }); + expect( + lockAllows(MODEL_LOCK, "agent-claude-sdk", "anthropic", "claude-opus-6"), + ).toBe(false); +}); + +test("what the check never ran is open, so nobody loses their models to missing data", () => { + // OpenBot's own runtime is no harness image. + expect(offered(undefined, "anthropic", anthropicIds)).toEqual(anthropicIds); + // No key for xAI in CI. + expect(lockVerdict(MODEL_LOCK, "agent-agno", "xai", "grok-4.7")).toEqual({ + status: "open", + }); + expect( + lockAllows(MODEL_LOCK, "agent-unknown", "openai", "gpt-6.1-terra"), + ).toBe(true); +}); + +test("every engine the lock names is a harness OpenBot publishes, with the clients it ran", async () => { + const { harnesses } = (await import("./harnesses.json")).default as { + harnesses: { image: string | null }[]; + }; + const images = harnesses.map((harness) => harness.image); + for (const [engine, entry] of Object.entries(MODEL_LOCK.engines)) { + expect(images, engine).toContain(engine); + expect(Object.keys(entry.clients).length, engine).toBeGreaterThan(0); + } +}); diff --git a/shared/model-lock.ts b/shared/model-lock.ts new file mode 100644 index 0000000000..b4f1d8e8ce --- /dev/null +++ b/shared/model-lock.ts @@ -0,0 +1,94 @@ +import manifest from "./model-lock.json"; + +/** + * THE MODEL LOCK: which models each harness has been proven to run, with the client versions it + * was proven on. + * + * WHY. The model picker lists each provider's models live, but every harness image carries its + * model client at a fixed version, and a provider can refuse a new model to an older client + * ("Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required"). A + * Bot set to such a model fails on every message. So a live model is offered only when the daily + * model check (scripts/model-compat.ts) has sent it one real request through that harness's + * own client and it answered. + * + * WHERE THE DATA COMES FROM. `model-lock.json` is written by `bun scripts/model-compat.ts lock` + * from the check's results, never by hand. The daily check opens a pull request when a model newly + * passes or starts failing, and a pull request that changes a harness's client checks that + * harness before it merges. See docs/releasing.md, "Model lock". + * + * THE RULE. A harness and provider pair the check has run is locked: only the models that passed + * are offered. A pair it has never run (no key for that provider in CI, or a harness it does not + * start) is open: nothing is known, so nothing is taken away. A model the provider lists that the + * check has not reached yet (released since the last run) is not offered until it passes. + */ + +/** Who serves the models, as the lock names them. `chatgpt` is a ChatGPT plan's Codex backend. */ +export type LockProvider = + | "openai" + | "anthropic" + | "google" + | "xai" + | "chatgpt"; + +export type LockedProvider = { + /** The model ids that answered through this harness's client. */ + verified: string[]; + /** The model ids the provider refused through this client, with what it said. */ + refused: Record; +}; + +export type LockedEngine = { + /** The model-client packages in the image the check ran, with their versions. */ + clients: Record; + providers: Partial>; +}; + +export type ModelLock = { + /** The day of the run the lock was written from, `YYYY-MM-DD`. */ + checkedAt: string; + /** By harness image directory (`agent-claude-sdk`), as `shared/harnesses.json` names it. */ + engines: Record; +}; + +export const MODEL_LOCK: ModelLock = manifest as ModelLock; + +/** + * What the lock says about one model on one harness and provider. + * + * - `verified`: it answered through the harness's own client. + * - `refused`: the provider turned it down through that client; `said` is the provider's words. + * - `unchecked`: the pair is locked, and this model was not in the run (released since). + * - `open`: the pair was never checked, or there is no harness to check (OpenBot's own runtime). + */ +export type LockVerdict = + | { status: "verified" } + | { status: "refused"; said: string } + | { status: "unchecked" } + | { status: "open" }; + +export function lockVerdict( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): LockVerdict { + const pair = + engine && provider ? lock.engines[engine]?.providers[provider] : undefined; + if (!pair) return { status: "open" }; + if (pair.verified.includes(model)) return { status: "verified" }; + const said = pair.refused[model]; + return said === undefined + ? { status: "unchecked" } + : { status: "refused", said }; +} + +/** Whether a model may be offered: proven on the pair, or the pair is not locked. */ +export function lockAllows( + lock: ModelLock, + engine: string | undefined, + provider: LockProvider | undefined, + model: string, +): boolean { + const { status } = lockVerdict(lock, engine, provider, model); + return status === "verified" || status === "open"; +} From a79bb1a66b0eff37d306b58101a76ccec7625990 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:28:57 -0800 Subject: [PATCH 07/18] Say an engine without an entry is open, not which one --- docs/releasing.md | 4 ++-- shared/model-lock.test.ts | 2 +- shared/model-lock.ts | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index f305456962..d1a0282dd5 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1135,8 +1135,8 @@ the harness is hidden, and a conversation already set to one runs on the default - **Which harness.** A Claude plan runs every Bot through `agent-claude-sdk`, a ChatGPT plan through `agent-langgraph-agui`, and with a key, the harness picked in setup when it takes OpenBot's model - choice. OpenBot's own runtime is not a harness image, so its models are not locked. -- **What is locked.** Only a harness and provider pair this check has run. A pair it never ran (a + choice. +- **What is locked.** Only an engine and provider pair that has been checked. A pair never checked (a provider with no key here, such as xAI today) stays open, and a gateway is never locked. - **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts index c9b638241d..3a34b91ecc 100644 --- a/shared/model-lock.test.ts +++ b/shared/model-lock.test.ts @@ -67,7 +67,7 @@ test("a model released after the last check waits for it", () => { }); test("what the check never ran is open, so nobody loses their models to missing data", () => { - // OpenBot's own runtime is no harness image. + // No engine named, or one the lock has no entry for. expect(offered(undefined, "anthropic", anthropicIds)).toEqual(anthropicIds); // No key for xAI in CI. expect(lockVerdict(MODEL_LOCK, "agent-agno", "xai", "grok-4.7")).toEqual({ diff --git a/shared/model-lock.ts b/shared/model-lock.ts index b4f1d8e8ce..9cb547f221 100644 --- a/shared/model-lock.ts +++ b/shared/model-lock.ts @@ -17,8 +17,8 @@ import manifest from "./model-lock.json"; * harness before it merges. See docs/releasing.md, "Model lock". * * THE RULE. A harness and provider pair the check has run is locked: only the models that passed - * are offered. A pair it has never run (no key for that provider in CI, or a harness it does not - * start) is open: nothing is known, so nothing is taken away. A model the provider lists that the + * are offered. A pair it has never run (no key for that provider in CI, or an engine it has no + * entry for) is open: nothing is known, so nothing is taken away. A model the provider lists that the * check has not reached yet (released since the last run) is not offered until it passes. */ @@ -58,7 +58,7 @@ export const MODEL_LOCK: ModelLock = manifest as ModelLock; * - `verified`: it answered through the harness's own client. * - `refused`: the provider turned it down through that client; `said` is the provider's words. * - `unchecked`: the pair is locked, and this model was not in the run (released since). - * - `open`: the pair was never checked, or there is no harness to check (OpenBot's own runtime). + * - `open`: the pair was never checked, or the lock has no entry for the engine. */ export type LockVerdict = | { status: "verified" } From ba3677a745d33269ad569a8d25cb50e25b2b8217 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:52:08 -0800 Subject: [PATCH 08/18] Write the model lock from the 2026-11-17 check, with Agno on main's fix Every published harness on OpenAI, Anthropic and Google, source images, run on a throwaway Linux box; agent-agno from main after its Claude fix. A rate limit is inconclusive, like a timeout, so a busy key never takes a model away. --- scripts/model-compat.ts | 6 +- scripts/model-lock.test.ts | 7 +- shared/model-lock.json | 1116 +++++++++++++++++++++++++++++++++++- 3 files changed, 1126 insertions(+), 3 deletions(-) diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index d0e28ec44c..462082afa7 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -1259,8 +1259,12 @@ export function providerIdOf(row: { const INCONCLUSIVE = /^(no answer:|the harness did not start|the harness never became healthy|the harness published no port|could not get |the stream ended without finishing the run)/; +/** A provider limiting how fast the check's key may call it: about the key's day, not the model. */ +const RATE_LIMITED = + /\b429\b|rate.?limit|RESOURCE_EXHAUSTED|too many requests/i; + export function inconclusive(detail: string): boolean { - return INCONCLUSIVE.test(detail); + return INCONCLUSIVE.test(detail) || RATE_LIMITED.test(detail); } function sortedRecord(record: Record): Record { diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index 705048f544..f7d625106a 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -81,7 +81,12 @@ test("a bad day for the runner takes nothing away", () => { "fail", "the harness never became healthy: exit 1", ), - row("claude-opus-5-5", "fail", "no answer: TimeoutError"), + // What Antigravity said for gemma-4-31b-it on 2026-11-17, while it answered on `latest`. + row( + "claude-opus-5-5", + "fail", + 'AntigravityExecutionError: Agent execution terminated due to error. ("agent executor error: Error 429, Message: Resource has been exhausted")', + ), ], }; const after = buildLock(before, [flaky], "2026-11-18"); diff --git a/shared/model-lock.json b/shared/model-lock.json index 6200ff35f2..8b37881a6b 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -1,4 +1,1118 @@ { "checkedAt": "2026-11-17", - "engines": {} + "engines": { + "agent-adk": { + "clients": { + "google-adk": "2.14.1", + "google-genai": "2.34.0", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", + "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-ag2": { + "clients": { + "ag2": "1.3.2", + "anthropic": "2.10.0", + "openai": "3.53.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "NotFoundError('Error code: 404')", + "gemini-decision-1": "NotFoundError('Error code: 404')", + "gemini-omni-1.1-flash": "BadRequestError('Error code: 400')", + "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5.1", + "gpt-5.2", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.5", + "o3" + ], + "refused": { + "gpt-5-pro": "NotFoundError('Error code: 404 - This model is only supported in v1/responses and not in v1/chat/completions.')", + "gpt-5.2-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", + "gpt-5.4-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", + "gpt-5.5-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", + "gpt-5.6-luna": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-5.6-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-5.6-terra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6-astra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6-luna": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6.1-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.1-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6.1-terra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.1-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6.2-astra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.2-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "gpt-6.2-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.2-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", + "o3-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')" + } + } + } + }, + "agent-agno": { + "clients": { + "agno": "3.2.1", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-2.5-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-2.5-pro": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3-flash-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-flash-lite-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-pro-preview": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.1-pro-preview-customtools": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.5-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.5-flash-lite": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.6-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.7-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-3.8-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-argon": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-carbon": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-4-hydrogen": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-decision-1": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-flash-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-flash-lite-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemini-pro-latest": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemma-4-26b-a4b-it": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei...", + "gemma-4-31b-it": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'Invalid JSON payload received. Unknown name \"external_execution\" at \\'tools[0].function\\': Cannot find field.', 'status': 'INVALID_ARGUMENT', 'details': [{'@type': 'type.googleapis.com/google.rpc.BadRequest', 'fieldViolations': [{'field': 'tools[0].function', 'description': 'Invalid JSON payload recei..." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-antigravity": { + "clients": { + "google-genai": "3.4.0" + }, + "providers": { + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it" + ], + "refused": { + "gemini-2.5-pro": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started)., Status: NOT_FOUND, De...", + "gemini-decision-1": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: models/gemini-decision-1 is not found for API version v1beta, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods., Status: NOT_FOUND, Details: []: Error 404, Message: models/gemini-decision-1 is not found ...", + "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", + "gemini-omni-1.2-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)" + } + } + } + }, + "agent-claude-sdk": { + "clients": { + "ag-ui-claude-sdk": "0.1.5", + "anthropic": "2.10.0", + "claude-agent-sdk": "0.2.152", + "claude-code": "2.1.259" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": { + "claude-fable-5-2": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.", + "claude-opus-5-5": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again." + } + } + } + }, + "agent-crewai": { + "clients": { + "crewai": "1.15.41", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "thread=model-compat-30ca7c4c01cd run=run-30ca7c4c01cd: CrewAI flow failed; see server logs", + "gemini-decision-1": "thread=model-compat-8ced77214a09 run=run-8ced77214a09: CrewAI flow failed; see server logs", + "gemini-omni-1.1-flash": "thread=model-compat-b983fa720574 run=run-b983fa720574: CrewAI flow failed; see server logs", + "gemini-omni-1.2-flash": "thread=model-compat-55d32045e810 run=run-55d32045e810: CrewAI flow failed; see server logs" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-langgraph-agui": { + "clients": { + "anthropic": "2.10.0", + "google-genai": "2.34.0", + "langchain-anthropic": "1.7.10", + "langchain-core": "1.8.1", + "langchain-google-genai": "4.4.3", + "langchain-openai": "1.8.1", + "langgraph": "1.2.22", + "openai": "3.53.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "chatgpt": { + "verified": [ + "gpt-5.5", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "Error code: 404", + "gemini-decision-1": "Error code: 404", + "gemini-omni-1.1-flash": "Error code: 400", + "gemini-omni-1.2-flash": "Error code: 400" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-langroid": { + "clients": { + "google-genai": "3.4.0", + "langroid": "0.69.24", + "litellm": "1.105.7", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'}}]", + "gemini-decision-1": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", + "gemini-omni-1.1-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5.1", + "gpt-5.2", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano" + ], + "refused": { + "gpt-5-pro": "Agent error: Error code: 404 - {'error': {'message': 'This model is only supported in v1/responses and not in v1/chat/completions.', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", + "gpt-5.2-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", + "gpt-5.4-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", + "gpt-5.5": "Agent error: Error code: 400 - {'error': {'message': \"Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported.\", 'type': 'invalid_request_error', 'param': 'temperature', 'code': 'unsupported_value'}}", + "gpt-5.5-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", + "gpt-5.6-luna": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-5.6-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-5.6-terra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6-astra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6-luna": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6.1-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.1-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6.1-terra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.1-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6.2-astra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.2-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "gpt-6.2-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.2-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", + "o3": "Agent error: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'parallel_tool_calls' is not supported with this model.\", 'type': 'invalid_request_error', 'param': 'parallel_tool_calls', 'code': 'unsupported_parameter'}}", + "o3-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}" + } + } + } + }, + "agent-llamaindex": { + "clients": { + "litellm": "1.105.7", + "llama-index-core": "0.14.26", + "openai": "2.54.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", + "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-mastra": { + "clients": { + "@ai-sdk/anthropic": "4.0.97", + "@ai-sdk/openai": "4.0.109", + "@mastra/core": "1.82.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", + "gemini-decision-1": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", + "gemini-omni-1.1-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", + "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd..." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-microsoft": { + "clients": { + "agent-framework-core": "1.26.0", + "anthropic": "1.17.0", + "openai": "3.53.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "An internal error has occurred while streaming events.", + "gemini-2.5-flash-lite": "An internal error has occurred while streaming events.", + "gemini-2.5-pro": "An internal error has occurred while streaming events.", + "gemini-3-flash-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-flash-lite": "An internal error has occurred while streaming events.", + "gemini-3.1-flash-lite-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-pro-preview": "An internal error has occurred while streaming events.", + "gemini-3.1-pro-preview-customtools": "An internal error has occurred while streaming events.", + "gemini-3.5-flash": "An internal error has occurred while streaming events.", + "gemini-3.5-flash-lite": "An internal error has occurred while streaming events.", + "gemini-3.6-flash": "An internal error has occurred while streaming events.", + "gemini-3.7-flash": "An internal error has occurred while streaming events.", + "gemini-3.8-flash": "An internal error has occurred while streaming events.", + "gemini-4-argon": "An internal error has occurred while streaming events.", + "gemini-4-carbon": "An internal error has occurred while streaming events.", + "gemini-4-hydrogen": "An internal error has occurred while streaming events.", + "gemini-decision-1": "An internal error has occurred while streaming events.", + "gemini-flash-latest": "An internal error has occurred while streaming events.", + "gemini-flash-lite-latest": "An internal error has occurred while streaming events.", + "gemini-omni-1.1-flash": "An internal error has occurred while streaming events.", + "gemini-omni-1.2-flash": "An internal error has occurred while streaming events.", + "gemini-pro-latest": "An internal error has occurred while streaming events.", + "gemma-4-26b-a4b-it": "An internal error has occurred while streaming events.", + "gemma-4-31b-it": "An internal error has occurred while streaming events." + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-pydantic-ai": { + "clients": { + "anthropic": "1.17.0", + "google-genai": "2.34.0", + "openai": "3.53.0", + "pydantic-ai-slim": "2.69.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [], + "refused": { + "gemini-2.5-flash": "status_code: 404, model_name: gemini-2.5-flash, body:", + "gemini-2.5-flash-lite": "status_code: 404, model_name: gemini-2.5-flash-lite, body:", + "gemini-2.5-pro": "status_code: 404, model_name: gemini-2.5-pro, body:", + "gemini-3-flash-preview": "status_code: 404, model_name: gemini-3-flash-preview, body:", + "gemini-3.1-flash-lite": "status_code: 404, model_name: gemini-3.1-flash-lite, body:", + "gemini-3.1-flash-lite-preview": "status_code: 404, model_name: gemini-3.1-flash-lite-preview, body:", + "gemini-3.1-pro-preview": "status_code: 404, model_name: gemini-3.1-pro-preview, body:", + "gemini-3.1-pro-preview-customtools": "status_code: 404, model_name: gemini-3.1-pro-preview-customtools, body:", + "gemini-3.5-flash": "status_code: 404, model_name: gemini-3.5-flash, body:", + "gemini-3.5-flash-lite": "status_code: 404, model_name: gemini-3.5-flash-lite, body:", + "gemini-3.6-flash": "status_code: 404, model_name: gemini-3.6-flash, body:", + "gemini-3.7-flash": "status_code: 404, model_name: gemini-3.7-flash, body:", + "gemini-3.8-flash": "status_code: 404, model_name: gemini-3.8-flash, body:", + "gemini-4-argon": "status_code: 404, model_name: gemini-4-argon, body:", + "gemini-4-carbon": "status_code: 404, model_name: gemini-4-carbon, body:", + "gemini-4-hydrogen": "status_code: 404, model_name: gemini-4-hydrogen, body:", + "gemini-decision-1": "status_code: 404, model_name: gemini-decision-1, body:", + "gemini-flash-latest": "status_code: 404, model_name: gemini-flash-latest, body:", + "gemini-flash-lite-latest": "status_code: 404, model_name: gemini-flash-lite-latest, body:", + "gemini-omni-1.1-flash": "status_code: 404, model_name: gemini-omni-1.1-flash, body:", + "gemini-omni-1.2-flash": "status_code: 404, model_name: gemini-omni-1.2-flash, body:", + "gemini-pro-latest": "status_code: 404, model_name: gemini-pro-latest, body:", + "gemma-4-26b-a4b-it": "status_code: 404, model_name: gemma-4-26b-a4b-it, body:", + "gemma-4-31b-it": "status_code: 404, model_name: gemma-4-31b-it, body:" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, + "agent-strands": { + "clients": { + "litellm": "1.105.7", + "openai": "2.54.0", + "strands-agents": "1.65.0" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + "gemma-4-26b-a4b-it", + "gemma-4-31b-it" + ], + "refused": { + "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", + "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", + "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + } + } } From 6f1193d5de952be5b9da2ef67e11f25d0443f474 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:59:34 -0800 Subject: [PATCH 09/18] Tell the client from the key: a model the key can't use takes nothing away Two Gemini keys disagreed on gemini-2.5-pro and gemini-2.5-flash-lite through the same clients, because Google closes older models to new projects. Such a failure, a bare 404 and a rate limit now keep what the lock knew, and the pull request check also runs the published image with the same keys, so a verified model counts as broken only when the change's image fails it and the published one doesn't. --- .github/workflows/model-lock.yml | 4 +- docs/releasing.md | 12 +++--- scripts/model-compat.ts | 33 +++++++++++++++- scripts/model-lock.test.ts | 64 ++++++++++++++++++++++++++++++++ shared/model-lock.json | 14 ------- 5 files changed, 106 insertions(+), 21 deletions(-) diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml index 0785135dee..4ef957b187 100644 --- a/.github/workflows/model-lock.yml +++ b/.github/workflows/model-lock.yml @@ -86,8 +86,10 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + # The published image too, with the same keys in the same run: a verified model that fails + # on both is the provider's or the key's today, not this change's (`brokenVerified`). run: >- - bun scripts/model-compat.ts --harness "$HARNESS" --tags source --exit-zero + bun scripts/model-compat.ts --harness "$HARNESS" --tags latest,source --exit-zero --out "$RUNNER_TEMP/model-compat.json" - name: Every model the lock verifies still answers env: diff --git a/docs/releasing.md b/docs/releasing.md index d1a0282dd5..7330878e85 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1141,13 +1141,15 @@ the harness is hidden, and a conversation already set to one runs on the default - **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that started and stopped answering and the clients that moved. Merging it lets the picker offer them. - A failure that says nothing about the model (the harness never started, a timeout) keeps what the - lock said before. + A failure that says nothing about the client keeps what the lock said before: the harness never + started, a timeout, a rate limit, or a model the check's key can't use (Google closes older models + to new projects, so two keys can disagree about the same model through the same client). - **A client bump** is held to the lock: a pull request from this repository that changes a harness's `requirements.txt`, `package.json`, `bun.lock`, `Dockerfile` or the Codex client version - runs the **Model lock** workflow, which builds that harness, sends every listed model a request, - and fails when a model the lock verifies no longer answers. Models the bump newly fixes are added - by the next daily refresh. + runs the **Model lock** workflow, which builds that harness, sends every listed model a request + through it and through the published image, and fails when a model the lock verifies no longer + answers through the change's image but still does through the published one. Models the bump + newly fixes are added by the next daily refresh. To write the lock from results by hand: diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index 462082afa7..dc494dd307 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -1263,8 +1263,23 @@ const INCONCLUSIVE = const RATE_LIMITED = /\b429\b|rate.?limit|RESOURCE_EXHAUSTED|too many requests/i; +/** + * A model this key's account can't use, which another account can: Google closes older models to + * new projects ("no longer available to new users") and answers a model a project can't see with a + * bare 404. Seen on 2026-11-17, when two Gemini keys disagreed on gemini-2.5-pro and + * gemini-2.5-flash-lite through the same clients. About the key, not the client. A 404 that names + * the client's endpoint ("only supported in v1/responses") is the client's, and stays a refusal. + */ +const UNAVAILABLE_TO_KEY = + /no longer available to new users|is not found for API version|does not exist or you do not have access|do not have access to (?:the )?model|^(?:NotFoundError\(')?Error code: 404'?\)?$|^Not Found$/i; + +/** A failure about the check's key, not the harness's client: a rate limit, or a model it can't see. */ +export function aboutTheKey(detail: string): boolean { + return RATE_LIMITED.test(detail) || UNAVAILABLE_TO_KEY.test(detail); +} + export function inconclusive(detail: string): boolean { - return INCONCLUSIVE.test(detail) || RATE_LIMITED.test(detail); + return INCONCLUSIVE.test(detail) || aboutTheKey(detail); } function sortedRecord(record: Record): Record { @@ -1413,8 +1428,24 @@ export function brokenVerified( lock: ModelLock, results: readonly Result[], ): Result[] { + /* + * What the published image did with the same key in the same run, when the run checked it too: a + * model that fails there as well is the provider's or the key's today, not this change's. + */ + const failsOnPublished = (row: Result) => + results.some( + (other) => + other.tag !== SOURCE && + other.status === "fail" && + other.harness === row.harness && + other.model === row.model && + providerIdOf(other) === providerIdOf(row), + ); return results.filter((row) => { if (row.status !== "fail" || row.model === "(all)") return false; + if (row.tag !== SOURCE) return false; + // The key's account, not the change's client: says nothing about the change. + if (aboutTheKey(row.detail) || failsOnPublished(row)) return false; const id = providerIdOf(row); return ( !!id && diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index f7d625106a..c1c5f268a4 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -4,6 +4,7 @@ import { brokenVerified, buildLock, describeLockChange, + inconclusive, providerIdOf, type Result, type Results, @@ -235,3 +236,66 @@ test("a client bump that breaks a verified model is caught, and one the lock alr ["claude-sonnet-5-5"], ); }); + +test("a model the check's key can't use is the key's, not the client's", () => { + const google = (model: string, status: Result["status"], detail = "OK") => + row(model, status, detail, { + harness: "agent-langgraph-agui", + provider: "Google", + providerId: "google", + }); + // One Gemini key's run on 2026-11-17; another key, through the same image, disagreed on both. + const lock = buildLock( + undefined, + [ + { + results: [ + google("gemini-2.5-flash-lite", "pass"), + google( + "gemini-2.5-pro", + "fail", + "Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users.'}}]", + ), + google( + "gemini-omni-1.1-flash", + "fail", + "Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.'}}]", + ), + ], + }, + ], + "2026-11-17", + ); + const pair = lock.engines["agent-langgraph-agui"]?.providers.google; + expect(pair?.verified).toEqual(["gemini-2.5-flash-lite"]); + // Not refused: another key may run it, and a later run with one proves it. + expect(Object.keys(pair?.refused ?? {})).toEqual(["gemini-omni-1.1-flash"]); + expect( + brokenVerified(lock, [ + google("gemini-2.5-flash-lite", "fail", "Error code: 404"), + ]), + ).toEqual([]); + // CrewAI hides the provider's words; the published image failing too, with the same key in the + // same run, says it isn't the change. + const hidden = () => + google( + "gemini-2.5-flash-lite", + "fail", + "CrewAI flow failed; see server logs", + ); + expect( + brokenVerified(lock, [hidden(), { ...hidden(), tag: "latest" }]), + ).toEqual([]); + expect( + brokenVerified(lock, [ + hidden(), + { ...google("gemini-2.5-flash-lite", "pass"), tag: "latest" }, + ]).map((broken) => broken.model), + ).toEqual(["gemini-2.5-flash-lite"]); + // A 404 that names the client's endpoint is the client's. + expect( + inconclusive( + "NotFoundError('Error code: 404 - This model is only supported in v1/responses and not in v1/chat/completions.')", + ), + ).toBe(false); +}); diff --git a/shared/model-lock.json b/shared/model-lock.json index 8b37881a6b..40763adfe5 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -53,8 +53,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", - "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" } @@ -146,8 +144,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "NotFoundError('Error code: 404')", - "gemini-decision-1": "NotFoundError('Error code: 404')", "gemini-omni-1.1-flash": "BadRequestError('Error code: 400')", "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')" } @@ -309,8 +305,6 @@ "gemma-4-26b-a4b-it" ], "refused": { - "gemini-2.5-pro": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started)., Status: NOT_FOUND, De...", - "gemini-decision-1": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: models/gemini-decision-1 is not found for API version v1beta, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods., Status: NOT_FOUND, Details: []: Error 404, Message: models/gemini-decision-1 is not found ...", "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", "gemini-omni-1.2-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)" } @@ -513,8 +507,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "Error code: 404", - "gemini-decision-1": "Error code: 404", "gemini-omni-1.1-flash": "Error code: 400", "gemini-omni-1.2-flash": "Error code: 400" } @@ -607,8 +599,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'}}]", - "gemini-decision-1": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", "gemini-omni-1.1-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" } @@ -701,8 +691,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", - "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" } @@ -1072,8 +1060,6 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users. Please update your code to use models/gemini-3.1-pro-preview for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'...", - "gemini-decision-1": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.', 'status': 'NOT_FOUND'}}]", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" } From bf42c10f37f891100bf7e8557d1a33d493ee5d25 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 14:21:59 -0800 Subject: [PATCH 10/18] Fail when the model lock was proven on other clients than the ones pinned The lock records each harness's clients; scripts/model-lock.test.ts now fails when a harness pins another version (an exact requirements pin, Mastra's bun.lock, the Codex client version), which is how a lock written before the Claude Code bump kept hiding Opus 5.5 and Fable 5.2. The pull request check uploads its results so the lock can be written from the change's own run. --- .github/workflows/model-lock.yml | 8 ++++ docs/releasing.md | 5 +++ scripts/model-compat.ts | 76 +++++++++++++++++++++++++++++++- scripts/model-lock.test.ts | 69 ++++++++++++++++++++++++++++- 4 files changed, 155 insertions(+), 3 deletions(-) diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml index 4ef957b187..5566b08f15 100644 --- a/.github/workflows/model-lock.yml +++ b/.github/workflows/model-lock.yml @@ -91,6 +91,14 @@ jobs: run: >- bun scripts/model-compat.ts --harness "$HARNESS" --tags latest,source --exit-zero --out "$RUNNER_TEMP/model-compat.json" + # The run's results, with the repository's own keys: `bun scripts/model-compat.ts lock` writes + # the lock from them when a pin moved (docs/releasing.md, "Model lock"). + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: model-lock-${{ matrix.harness }} + path: ${{ runner.temp }}/model-compat.json + if-no-files-found: error + retention-days: 14 - name: Every model the lock verifies still answers env: HARNESS: ${{ matrix.harness }} diff --git a/docs/releasing.md b/docs/releasing.md index 7330878e85..7c39b9fecf 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1150,6 +1150,11 @@ the harness is hidden, and a conversation already set to one runs on the default through it and through the published image, and fails when a model the lock verifies no longer answers through the change's image but still does through the published one. Models the bump newly fixes are added by the next daily refresh. +- **The lock matches the pins.** The lock records the clients it was proven on, and + `scripts/model-lock.test.ts` fails when a harness pins a different version (an exact + `requirements.txt` pin, `agent-mastra/bun.lock`, or the Codex client version): a lock proven on + other clients may hide models that now work. Write it again from the change's own **Model lock** + run, whose results each harness uploads as `model-lock-`: download them and run `lock`. To write the lock from results by hand: diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index dc494dd307..2fa35dbdb7 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -796,7 +796,7 @@ async function prepare(harness: Harness, tag: string) { * ecosystem installs them under. Whichever an image has are recorded in the lock beside the * models they were proven with, so a lock entry says what it was true for. */ -const PYTHON_CLIENTS = [ +export const PYTHON_CLIENTS = [ "claude-agent-sdk", "ag-ui-claude-sdk", "openai", @@ -808,6 +808,7 @@ const PYTHON_CLIENTS = [ "agno", "crewai", "langgraph", + "ag-ui-langgraph", "langchain-core", "langchain-openai", "langchain-anthropic", @@ -822,7 +823,7 @@ const PYTHON_CLIENTS = [ "agent-framework", "agent-framework-core", ]; -const NODE_CLIENTS = [ +export const NODE_CLIENTS = [ "@mastra/core", "ai", "@ai-sdk/openai", @@ -1167,6 +1168,12 @@ async function check(argv: string[]): Promise { ]); if (version.code !== 0) continue; const sends = version.stdout.trim(); + // What the plan's backend is told, recorded with the image's other clients. + if (harness.image) + clients[harness.image] = { + ...clients[harness.image], + [tag]: { ...clients[harness.image]?.[tag], "codex-client": sends }, + }; if (!codex.ok) { results.push({ harness: harness.image ?? "", @@ -1423,6 +1430,71 @@ export function describeLockChange( return lines.length ? lines.join("\n") : "No model changed."; } +/** + * The model-client versions a harness pins in this repository, by the names the lock records + * them under: exact `name==version` lines in its `requirements.txt`, the versions its `bun.lock` + * resolves for the Node clients, and the Codex client version the LangGraph harness sends a + * ChatGPT plan. Unpinned packages are left out: only what the repository fixes can disagree. + */ +export function pinnedClients( + root: string, + harness: string, +): Record { + const pinned: Record = {}; + const read = (file: string) => { + const path = join(root, harness, file); + return existsSync(path) ? readFileSync(path, "utf8") : ""; + }; + for (const line of read("requirements.txt").split("\n")) { + const match = /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?\s*==\s*([^\s;#]+)/.exec( + line, + ); + if (match?.[1] && match[2]) + pinned[match[1].toLowerCase().replace(/_/g, "-")] = match[2]; + } + const lockfile = read("bun.lock"); + for (const name of NODE_CLIENTS) { + const escaped = name.replace(/[/.]/g, "\\$&"); + const found = new RegExp(`"${escaped}": \\["${escaped}@([^"]+)"`).exec( + lockfile, + )?.[1]; + if (found) pinned[name] = found; + } + const codex = /^DEFAULT_CLIENT_VERSION\s*=\s*"([^"]+)"/m.exec( + read("src/chatgpt_models.py"), + )?.[1]; + if (codex) pinned["codex-client"] = codex; + return pinned; +} + +/** + * Where the lock and the repository disagree about a harness's clients: each pinned client the + * lock recorded at another version, or didn't record. A lock that disagrees was proven on other + * clients than the ones this commit ships, so it may hide models that now work, or offer ones that + * no longer do. Regenerate it (`lock`) from a run of this commit. + */ +export function staleClients( + lock: ModelLock, + root: string, +): { harness: string; client: string; pinned: string; locked?: string }[] { + const stale: { + harness: string; + client: string; + pinned: string; + locked?: string; + }[] = []; + for (const [harness, entry] of Object.entries(lock.engines)) { + for (const [client, pinned] of Object.entries( + pinnedClients(root, harness), + )) { + const locked = entry.clients[client]; + if (locked !== pinned) + stale.push({ harness, client, pinned, ...(locked ? { locked } : {}) }); + } + } + return stale; +} + /** Every model the lock verifies that now fails: what a client bump must not break. */ export function brokenVerified( lock: ModelLock, diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index c1c5f268a4..e796077e72 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -1,15 +1,82 @@ import { expect, test } from "bun:test"; -import type { ModelLock } from "../shared/model-lock"; +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { MODEL_LOCK, type ModelLock } from "../shared/model-lock"; import { brokenVerified, buildLock, describeLockChange, inconclusive, + PYTHON_CLIENTS, + pinnedClients, providerIdOf, type Result, type Results, + staleClients, } from "./model-compat"; +const ROOT = join(import.meta.dir, ".."); + +/* + * THE LOCK MATCHES WHAT THIS COMMIT SHIPS. Written on a commit whose Claude Agent SDK harness + * pinned claude-agent-sdk 0.2.152 (Claude Code 2.1.259), the lock refused Opus 5.5 and Fable 5.2; + * main then bumped the pin to a Claude Code that runs both, and the lock kept hiding them. A lock + * proven on other clients than the ones pinned here is stale, and fails here until it is written + * again from a run of this commit. + */ +test("the lock was proven on the model clients each harness pins in this commit", () => { + expect(staleClients(MODEL_LOCK, ROOT)).toEqual([]); +}); + +test("every exact pin in a harness's requirements is a client the check records", () => { + const harnesses = readdirSync(ROOT).filter((name) => + name.startsWith("agent-"), + ); + for (const harness of harnesses) { + let text = ""; + try { + text = readFileSync(join(ROOT, harness, "requirements.txt"), "utf8"); + } catch { + continue; + } + for (const match of text.matchAll( + /^\s*([A-Za-z0-9_.-]+)(?:\[[^\]]*\])?\s*==/gm, + )) + expect(PYTHON_CLIENTS, `${harness}: ${match[1]}`).toContain( + (match[1] ?? "").toLowerCase().replace(/_/g, "-"), + ); + } +}); + +test("a pin that moved past the lock is caught", () => { + const claude = pinnedClients(ROOT, "agent-claude-sdk"); + expect(Object.keys(claude)).toContain("claude-agent-sdk"); + const old: ModelLock = { + checkedAt: "2026-11-17", + engines: { + "agent-claude-sdk": { + clients: { ...claude, "claude-agent-sdk": "0.2.152" }, + providers: {}, + }, + }, + }; + expect(staleClients(old, ROOT)).toEqual([ + { + harness: "agent-claude-sdk", + client: "claude-agent-sdk", + pinned: claude["claude-agent-sdk"] ?? "", + locked: "0.2.152", + }, + ]); + // The Codex version the LangGraph harness sends a ChatGPT plan, and Mastra's resolved clients. + expect(Object.keys(pinnedClients(ROOT, "agent-langgraph-agui"))).toContain( + "codex-client", + ); + expect(Object.keys(pinnedClients(ROOT, "agent-mastra"))).toContain( + "@mastra/core", + ); +}); + /* * Writing the model lock from the model check's results, and holding a client bump to it. The * rows are what the check reported on 2026-11-17, in its own words. From f1d1db62dd9d76bddc37687b1b8b1c2d23688402 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 14:32:38 -0800 Subject: [PATCH 11/18] Write the model lock from this branch's own run on main's pins From the Model lock run on 11032611b, with the repository's keys: the Claude Agent SDK harness on claude-agent-sdk 0.2.182 (Claude Code 2.1.325) answers on all 15 Anthropic models, Opus 5.5 and Fable 5.2 included, and AG2 and Langroid answer on all 29 OpenAI models. The lock from before the bump stays as a test fixture for the rule. --- .../model-lock-before-claude-code-bump.json | 36 ++++++ shared/model-lock.json | 117 +++++++++--------- shared/model-lock.test.ts | 80 +++++++++--- 3 files changed, 156 insertions(+), 77 deletions(-) create mode 100644 shared/fixtures/model-lock-before-claude-code-bump.json diff --git a/shared/fixtures/model-lock-before-claude-code-bump.json b/shared/fixtures/model-lock-before-claude-code-bump.json new file mode 100644 index 0000000000..4bbabc631d --- /dev/null +++ b/shared/fixtures/model-lock-before-claude-code-bump.json @@ -0,0 +1,36 @@ +{ + "checkedAt": "2026-11-17", + "engines": { + "agent-claude-sdk": { + "clients": { + "ag-ui-claude-sdk": "0.1.5", + "anthropic": "2.10.0", + "claude-agent-sdk": "0.2.152", + "claude-code": "2.1.259" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": { + "claude-fable-5-2": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.", + "claude-opus-5-5": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again." + } + } + } + } + } +} diff --git a/shared/model-lock.json b/shared/model-lock.json index 40763adfe5..0b1b2174ee 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -32,7 +32,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -123,7 +123,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -157,31 +157,30 @@ "gpt-5", "gpt-5-mini", "gpt-5-nano", + "gpt-5-pro", "gpt-5.1", "gpt-5.2", + "gpt-5.2-pro", "gpt-5.4", "gpt-5.4-mini", "gpt-5.4-nano", + "gpt-5.4-pro", "gpt-5.5", - "o3" + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" ], - "refused": { - "gpt-5-pro": "NotFoundError('Error code: 404 - This model is only supported in v1/responses and not in v1/chat/completions.')", - "gpt-5.2-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", - "gpt-5.4-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", - "gpt-5.5-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')", - "gpt-5.6-luna": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-5.6-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-5.6-terra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-5.6-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6-astra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6-luna": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6.1-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.1-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6.1-terra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.1-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6.2-astra": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.2-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "gpt-6.2-sol": "BadRequestError(\"Error code: 400 - Function tools with reasoning_effort are not supported for gpt-6.2-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\")", - "o3-pro": "NotFoundError('Error code: 404 - This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?')" - } + "refused": {} } } }, @@ -285,7 +284,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -302,7 +301,7 @@ "gemini-flash-latest", "gemini-flash-lite-latest", "gemini-pro-latest", - "gemma-4-26b-a4b-it" + "gemma-4-31b-it" ], "refused": { "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", @@ -315,14 +314,15 @@ "clients": { "ag-ui-claude-sdk": "0.1.5", "anthropic": "2.10.0", - "claude-agent-sdk": "0.2.152", - "claude-code": "2.1.259" + "claude-agent-sdk": "0.2.182", + "claude-code": "2.1.325" }, "providers": { "anthropic": { "verified": [ "claude-fable-5", "claude-fable-5-1", + "claude-fable-5-2", "claude-haiku-4-5-20251001", "claude-haiku-5-5", "claude-opus-4-5-20251101", @@ -330,15 +330,13 @@ "claude-opus-4-7", "claude-opus-4-8", "claude-opus-5", + "claude-opus-5-5", "claude-sonnet-4-5-20250929", "claude-sonnet-4-6", "claude-sonnet-5", "claude-sonnet-5-5" ], - "refused": { - "claude-fable-5-2": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.290 or newer is required. Run 'claude update', or update the Claude desktop app, then try again.", - "claude-opus-5-5": "API Error: 400 Claude Code 2.1.259 does not support this model; version 2.1.280 or newer is required. Run 'claude update', or update the Claude desktop app, then try again." - } + "refused": {} } } }, @@ -372,7 +370,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -393,10 +391,10 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "thread=model-compat-30ca7c4c01cd run=run-30ca7c4c01cd: CrewAI flow failed; see server logs", - "gemini-decision-1": "thread=model-compat-8ced77214a09 run=run-8ced77214a09: CrewAI flow failed; see server logs", - "gemini-omni-1.1-flash": "thread=model-compat-b983fa720574 run=run-b983fa720574: CrewAI flow failed; see server logs", - "gemini-omni-1.2-flash": "thread=model-compat-55d32045e810 run=run-55d32045e810: CrewAI flow failed; see server logs" + "gemini-2.5-flash-lite": "thread=model-compat-21c5bc66a9c4 run=run-21c5bc66a9c4: CrewAI flow failed; see server logs", + "gemini-decision-1": "thread=model-compat-582497527ff7 run=run-582497527ff7: CrewAI flow failed; see server logs", + "gemini-omni-1.1-flash": "thread=model-compat-ce043e6f69ce run=run-ce043e6f69ce: CrewAI flow failed; see server logs", + "gemini-omni-1.2-flash": "thread=model-compat-702a8e9c82e6 run=run-702a8e9c82e6: CrewAI flow failed; see server logs" } }, "openai": { @@ -437,7 +435,9 @@ }, "agent-langgraph-agui": { "clients": { + "ag-ui-langgraph": "0.0.45", "anthropic": "2.10.0", + "codex-client": "0.167.0", "google-genai": "2.34.0", "langchain-anthropic": "1.7.10", "langchain-core": "1.8.1", @@ -486,7 +486,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -578,7 +578,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -612,31 +612,30 @@ "gpt-5", "gpt-5-mini", "gpt-5-nano", + "gpt-5-pro", "gpt-5.1", "gpt-5.2", + "gpt-5.2-pro", "gpt-5.4", "gpt-5.4-mini", - "gpt-5.4-nano" + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" ], - "refused": { - "gpt-5-pro": "Agent error: Error code: 404 - {'error': {'message': 'This model is only supported in v1/responses and not in v1/chat/completions.', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", - "gpt-5.2-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", - "gpt-5.4-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", - "gpt-5.5": "Agent error: Error code: 400 - {'error': {'message': \"Unsupported value: 'temperature' does not support 0.2 with this model. Only the default (1) value is supported.\", 'type': 'invalid_request_error', 'param': 'temperature', 'code': 'unsupported_value'}}", - "gpt-5.5-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}", - "gpt-5.6-luna": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-5.6-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-5.6-terra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-5.6-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6-astra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6-luna": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-luna in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6.1-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.1-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6.1-terra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.1-terra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6.2-astra": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.2-astra in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "gpt-6.2-sol": "Agent error: Error code: 400 - {'error': {'message': \"Function tools with reasoning_effort are not supported for gpt-6.2-sol in /v1/chat/completions. To use function tools, use /v1/responses or set reasoning_effort to 'none'.\", 'type': 'invalid_request_error', 'param': 'reasoning_effort', 'code': None}}", - "o3": "Agent error: Error code: 400 - {'error': {'message': \"Unsupported parameter: 'parallel_tool_calls' is not supported with this model.\", 'type': 'invalid_request_error', 'param': 'parallel_tool_calls', 'code': 'unsupported_parameter'}}", - "o3-pro": "Agent error: Error code: 404 - {'error': {'message': 'This is not a chat model and thus not supported in the v1/chat/completions endpoint. Did you mean to use v1/completions?', 'type': 'invalid_request_error', 'param': 'model', 'code': None}}" - } + "refused": {} } } }, @@ -670,7 +669,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -761,7 +760,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", @@ -782,7 +781,7 @@ "gemma-4-31b-it" ], "refused": { - "gemini-2.5-pro": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", + "gemini-2.5-flash-lite": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", "gemini-decision-1": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", "gemini-omni-1.1-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd..." @@ -1039,7 +1038,7 @@ "google": { "verified": [ "gemini-2.5-flash", - "gemini-2.5-flash-lite", + "gemini-2.5-pro", "gemini-3-flash-preview", "gemini-3.1-flash-lite", "gemini-3.1-flash-lite-preview", diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts index 3a34b91ecc..52d96f72cc 100644 --- a/shared/model-lock.test.ts +++ b/shared/model-lock.test.ts @@ -1,39 +1,46 @@ import { expect, test } from "bun:test"; import anthropicListing from "./fixtures/model-listings/anthropic-models.json"; import openaiListing from "./fixtures/model-listings/openai-models.json"; -import { lockAllows, lockVerdict, MODEL_LOCK } from "./model-lock"; +import before from "./fixtures/model-lock-before-claude-code-bump.json"; +import { + lockAllows, + lockVerdict, + MODEL_LOCK, + type ModelLock, +} from "./model-lock"; /* * The lock's rule against real data: the model listings OpenAI and Anthropic answered on - * 2026-11-17 (shared/fixtures/model-listings), and the lock the model check wrote that day. The - * Claude plan's failure that day was the Claude Agent SDK harness offering Opus 5.5 and Fable - * 5.2, which its pinned Claude Code was refused on every message. + * 2026-11-17 (shared/fixtures/model-listings), and two locks written that day. `before` is the + * Claude Agent SDK harness's entry from a run of claude-agent-sdk 0.2.152 (Claude Code 2.1.259), + * which Anthropic refused Opus 5.5 and Fable 5.2: every message a Claude plan sent them failed. + * `MODEL_LOCK` is the lock this commit ships, written from a run of its own pins. */ +const BEFORE = before as ModelLock; const anthropicIds = anthropicListing.data.map((model) => model.id); const openaiIds = openaiListing.data.map((model) => model.id); const offered = ( + lock: ModelLock, engine: string | undefined, provider: "anthropic" | "openai", ids: string[], -) => ids.filter((id) => lockAllows(MODEL_LOCK, engine, provider, id)); +) => ids.filter((id) => lockAllows(lock, engine, provider, id)); -test("the Claude Agent SDK harness is not offered the models its Claude Code is refused", () => { - const claude = offered("agent-claude-sdk", "anthropic", anthropicIds); +test("a harness is not offered the models its client is refused", () => { + const claude = offered(BEFORE, "agent-claude-sdk", "anthropic", anthropicIds); expect(anthropicIds).toContain("claude-opus-5-5"); expect(anthropicIds).toContain("claude-fable-5-2"); expect(claude).not.toContain("claude-opus-5-5"); expect(claude).not.toContain("claude-fable-5-2"); expect(claude).toContain("claude-sonnet-5-5"); - expect(claude).toContain("claude-haiku-5-5"); - // Everything else Anthropic listed that day answered. expect(claude).toHaveLength(anthropicIds.length - 2); }); test("a refused model carries the provider's own words", () => { const verdict = lockVerdict( - MODEL_LOCK, + BEFORE, "agent-claude-sdk", "anthropic", "claude-opus-5-5", @@ -44,17 +51,52 @@ test("a refused model carries the provider's own words", () => { ); }); -test("a harness whose client takes every model is offered every model", () => { - expect(offered("agent-langgraph-agui", "anthropic", anthropicIds)).toEqual( +test("once the pinned client runs them, the shipped lock offers them", () => { + // The Claude Code bump this commit pins. + expect( + MODEL_LOCK.engines["agent-claude-sdk"]?.clients["claude-code"], + ).not.toBe("2.1.259"); + const claude = offered( + MODEL_LOCK, + "agent-claude-sdk", + "anthropic", anthropicIds, ); + expect(claude).toContain("claude-opus-5-5"); + expect(claude).toContain("claude-fable-5-2"); }); -test("the Responses-only OpenAI models are not offered on a Chat Completions harness", () => { - const ag2 = offered("agent-ag2", "openai", openaiIds); - for (const model of ["gpt-5-pro", "o3-pro"]) - if (openaiIds.includes(model)) expect(ag2).not.toContain(model); - expect(ag2.length).toBeGreaterThan(0); +test("the shipped lock still holds back what a provider really refuses", () => { + // Google takes gemini-omni-1.x-flash only through its Interactions API, so no client here runs it. + const verdict = lockVerdict( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-omni-1.1-flash", + ); + expect(verdict.status).toBe("refused"); + expect( + lockAllows( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-omni-1.1-flash", + ), + ).toBe(false); + expect( + lockAllows( + MODEL_LOCK, + "agent-langgraph-agui", + "google", + "gemini-3.8-flash", + ), + ).toBe(true); +}); + +test("a harness that takes OpenAI's newest models through the Responses API is offered them", () => { + const ag2 = offered(MODEL_LOCK, "agent-ag2", "openai", openaiIds); + for (const model of ["gpt-6.1-terra", "gpt-5-pro"]) + if (openaiIds.includes(model)) expect(ag2).toContain(model); }); test("a model released after the last check waits for it", () => { @@ -68,7 +110,9 @@ test("a model released after the last check waits for it", () => { test("what the check never ran is open, so nobody loses their models to missing data", () => { // No engine named, or one the lock has no entry for. - expect(offered(undefined, "anthropic", anthropicIds)).toEqual(anthropicIds); + expect(offered(MODEL_LOCK, undefined, "anthropic", anthropicIds)).toEqual( + anthropicIds, + ); // No key for xAI in CI. expect(lockVerdict(MODEL_LOCK, "agent-agno", "xai", "grok-4.7")).toEqual({ status: "open", From a0b6b681a988155901f05d4c0a7496a98f5285c6 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 13:47:21 -0800 Subject: [PATCH 12/18] Lock the LangGraph (TypeScript) engine before its first release, and check it on every provider shared/model-lock.json names agent-langgraph-ts with empty pairs for Anthropic, OpenAI and Google, so nothing is offered on it until the model check has sent a model a real request through its own client and it answered, including on a provider CI has no key for. The check now runs every engine the lock names as well as the published harnesses (scripts/model-compat.ts harnesses, which both workflows' matrices use), so the engine is checked from its source image before it is published. The probe also reads the engine's LangChain and AG-UI client versions, under plain Node when an image has no Bun. --- .github/workflows/model-compat.yml | 9 +- .github/workflows/model-lock.yml | 14 +- docs/releasing.md | 5 + scripts/model-compat-langgraph-ts.test.ts | 335 ++++++++++++++++++++++ scripts/model-compat.ts | 97 +++++-- shared/model-lock.test.ts | 10 +- 6 files changed, 448 insertions(+), 22 deletions(-) create mode 100644 scripts/model-compat-langgraph-ts.test.ts diff --git a/.github/workflows/model-compat.yml b/.github/workflows/model-compat.yml index f41029ae1f..1db03a8315 100644 --- a/.github/workflows/model-compat.yml +++ b/.github/workflows/model-compat.yml @@ -36,8 +36,10 @@ concurrency: cancel-in-progress: false jobs: + # The published harnesses, and every engine the model lock names whether it is published yet or + # not (an engine new Bots run on is locked, and checked, from before its first release). harnesses: - name: List the published harnesses + name: List the harnesses to check if: github.repository == 'CopilotKit/OpenBot' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest timeout-minutes: 5 @@ -49,8 +51,11 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 - id: list - run: echo "images=$(jq -c . .github/published-images.json)" >> "$GITHUB_OUTPUT" + run: echo "images=$(bun scripts/model-compat.ts harnesses)" >> "$GITHUB_OUTPUT" check: name: ${{ matrix.harness }} diff --git a/.github/workflows/model-lock.yml b/.github/workflows/model-lock.yml index 5566b08f15..12f568d454 100644 --- a/.github/workflows/model-lock.yml +++ b/.github/workflows/model-lock.yml @@ -16,6 +16,7 @@ on: - "agent-*/Dockerfile" - "agent-*/package.json" - "agent-*/bun.lock" + - "agent-*/package-lock.json" - "agent-langgraph-agui/src/chatgpt_models.py" - "shared/model-lock.json" - "scripts/model-compat.ts" @@ -42,6 +43,9 @@ jobs: with: persist-credentials: false fetch-depth: 0 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: 1.4.3 - id: list env: BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -49,13 +53,17 @@ jobs: run: | set -euo pipefail changed=$(git diff --name-only "$BASE_SHA...$HEAD_SHA") + # The harnesses the check runs: the published ones, and every engine the lock names, + # published yet or not, that has a row in shared/harnesses.json. + checked=$(bun scripts/model-compat.ts harnesses) # The check itself, or the lock, changed: every harness the lock names is checked. if grep -qxE 'shared/model-lock.json|scripts/model-compat.ts|\.github/workflows/model-lock.yml' <<<"$changed"; then - harnesses=$(jq -c '[.engines | keys[]]' shared/model-lock.json) + harnesses=$(jq -c --argjson checked "$checked" \ + '[.engines | keys[] | select(. as $h | $checked | index($h))]' shared/model-lock.json) else harnesses=$(grep -oE '^agent-[a-z0-9-]+/' <<<"$changed" | tr -d / | sort -u \ - | jq -R . | jq -sc --slurpfile published .github/published-images.json \ - 'map(select(. as $h | $published[0] | index($h)))') + | jq -R . | jq -sc --argjson checked "$checked" \ + 'map(select(. as $h | $checked | index($h)))') fi echo "harnesses=$harnesses" >> "$GITHUB_OUTPUT" echo "Harnesses to check: $harnesses" diff --git a/docs/releasing.md b/docs/releasing.md index 7c39b9fecf..2c37aed788 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1138,6 +1138,11 @@ the harness is hidden, and a conversation already set to one runs on the default choice. - **What is locked.** Only an engine and provider pair that has been checked. A pair never checked (a provider with no key here, such as xAI today) stays open, and a gateway is never locked. +- **A new engine** can be locked before its first release, by giving it empty pairs for the + providers it runs on. Nothing is offered on it until a run proves a model, including on a + provider the check has no key for. The check runs every engine the lock names, published or not + yet, from its `source` image (`bun scripts/model-compat.ts harnesses` lists them). The LangGraph + (TypeScript) engine, `agent-langgraph-ts`, starts this way on Anthropic, OpenAI and Google. - **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that started and stopped answering and the clients that moved. Merging it lets the picker offer them. diff --git a/scripts/model-compat-langgraph-ts.test.ts b/scripts/model-compat-langgraph-ts.test.ts new file mode 100644 index 0000000000..0f84162835 --- /dev/null +++ b/scripts/model-compat-langgraph-ts.test.ts @@ -0,0 +1,335 @@ +import { expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import anthropicListing from "../shared/fixtures/model-listings/anthropic-models.json"; +import openaiListing from "../shared/fixtures/model-listings/openai-models.json"; +import { lockAllows, MODEL_LOCK, type ModelLock } from "../shared/model-lock"; +import { + buildLock, + buildMatrix, + type Harness, + NODE_VERSIONS, + probedHarnesses, + providersFor, + type Result, + type Results, + usableModels, +} from "./model-compat"; + +/* + * The LangGraph (TypeScript) engine, `agent-langgraph-ts`, is locked before its first release: + * it becomes the engine new Bots run on, so a model is offered on it only once the model check has + * sent that model a real request through the engine's own client and it answered. Until then, + * nothing is offered on it, for any provider, even one CI has no key for. + */ + +const ENGINE = "agent-langgraph-ts"; +const ROOT = join(import.meta.dir, ".."); +const NOW = Date.parse("2026-11-17T00:00:00Z"); + +/** Its row in shared/harnesses.json, as the shared contract names it. */ +const LANGGRAPH_TS: Harness & { id: string } = { + id: "langgraph-ts", + image: ENGINE, + port: 4215, + runPath: "", + credential: "any-provider", +}; + +const anthropicIds = usableModels( + "anthropic", + anthropicListing, + new Map(), + NOW, +); +const openaiIds = usableModels("openai", openaiListing, new Map(), NOW); +const geminiIds = ["gemini-3.8-flash", "gemini-4-argon", "gemini-3.5-pro"]; + +test("before its first run, nothing is offered on langgraph-ts, on any provider", () => { + expect(anthropicIds.length).toBeGreaterThan(0); + expect(openaiIds.length).toBeGreaterThan(0); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + lockAllows(MODEL_LOCK, ENGINE, provider, model), + `${provider} ${model}`, + ).toBe(false); +}); + +const row = ( + providerId: "anthropic" | "openai" | "google", + model: string, + status: Result["status"], + detail = "OK", + tag = "source", +): Result => ({ + harness: ENGINE, + tag, + provider: { anthropic: "Anthropic", openai: "OpenAI", google: "Google" }[ + providerId + ], + providerId, + model, + status, + detail, +}); + +test("after a run, exactly the models that answered are offered on langgraph-ts", () => { + const run: Results = { + results: [ + row("anthropic", "claude-sonnet-5-5", "pass"), + row("anthropic", "claude-opus-5-5", "pass"), + row( + "anthropic", + "claude-fable-5-2", + "fail", + "400 invalid_request_error: model: claude-fable-5-2", + ), + // The runner's bad day: says nothing about the model, and verifies nothing. + row( + "anthropic", + "claude-sonnet-4-5-20250929", + "fail", + "no answer: TimeoutError: The operation timed out.", + ), + row("openai", "gpt-6.1-terra", "pass"), + row( + "openai", + "gpt-5.5", + "fail", + "the stream ended without finishing the run", + ), + // The published image is what people ran before; the lock follows `source` alone. + row("openai", "gpt-6.2-sol", "pass", "OK", "latest"), + { + harness: ENGINE, + tag: "latest, source", + provider: "Google", + providerId: "google", + model: "(all)", + status: "skipped", + detail: "skipped: no key (GEMINI_API_KEY)", + }, + ], + clients: { + [ENGINE]: { + source: { + "@langchain/langgraph": "1.7.5", + "@langchain/anthropic": "1.3.1", + }, + }, + }, + }; + const lock = buildLock(MODEL_LOCK, [run], "2026-11-17"); + const allowed = ( + provider: "anthropic" | "openai" | "google", + ids: string[], + ) => ids.filter((model) => lockAllows(lock, ENGINE, provider, model)); + + expect(allowed("anthropic", anthropicIds).sort()).toEqual([ + "claude-opus-5-5", + "claude-sonnet-5-5", + ]); + expect(openaiIds).toContain("gpt-6.1-terra"); + expect(allowed("openai", openaiIds)).toEqual(["gpt-6.1-terra"]); + // No key for Gemini today: the pair stays locked, and nothing is offered on it. + expect(allowed("google", geminiIds)).toEqual([]); + expect(lock.engines[ENGINE]?.providers.google).toEqual({ + verified: [], + refused: {}, + }); + expect(lock.engines[ENGINE]?.providers.anthropic?.refused).toEqual({ + "claude-fable-5-2": "400 invalid_request_error: model: claude-fable-5-2", + }); + expect(lock.engines[ENGINE]?.clients).toEqual({ + "@langchain/anthropic": "1.3.1", + "@langchain/langgraph": "1.7.5", + }); +}); + +test("an inconclusive failure never verifies a model langgraph-ts has not answered", () => { + const before: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: { "@langchain/langgraph": "1.7.5" }, + providers: { + anthropic: { verified: ["claude-sonnet-5-5"], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, + }; + const inconclusive = [ + "no answer: TimeoutError: The operation timed out.", + "the harness did not start: port is already allocated", + "the harness never became healthy: Error: Cannot find module", + `could not get ghcr.io/copilotkit/openbot-${ENGINE}:latest: manifest unknown`, + "the stream ended without finishing the run", + ]; + const run: Results = { + results: inconclusive.flatMap((detail, index) => [ + row("anthropic", `claude-model-${index}`, "fail", detail), + row("openai", `gpt-model-${index}`, "fail", detail), + // One it had answered before keeps its place: a bad day takes nothing away. + row("anthropic", "claude-sonnet-5-5", "fail", detail), + ]), + }; + const lock = buildLock(before, [run], "2026-11-17"); + expect(lock.engines[ENGINE]?.providers.anthropic?.verified).toEqual([ + "claude-sonnet-5-5", + ]); + expect(lock.engines[ENGINE]?.providers.openai?.verified).toEqual([]); + for (let index = 0; index < inconclusive.length; index++) { + expect(lockAllows(lock, ENGINE, "anthropic", `claude-model-${index}`)).toBe( + false, + ); + expect(lockAllows(lock, ENGINE, "openai", `gpt-model-${index}`)).toBe( + false, + ); + } +}); + +test("the lock names langgraph-ts, with every provider a key reaches locked", () => { + const entry = MODEL_LOCK.engines[ENGINE]; + expect(entry).toBeDefined(); + for (const provider of ["anthropic", "openai", "google"] as const) + expect(entry?.providers[provider], provider).toBeDefined(); +}); + +test("an engine the lock names is checked before it is published", () => { + const published = ["agent-claude-sdk", "agent-langgraph-agui"]; + const known: Harness[] = [ + { + image: "agent-claude-sdk", + credential: "anthropic", + port: 4212, + runPath: "", + }, + { + image: "agent-langgraph-agui", + credential: "any-provider", + port: 4201, + runPath: "", + }, + LANGGRAPH_TS, + { + image: "agent-unlisted", + credential: "any-provider", + port: 4299, + runPath: "", + }, + ]; + const lock: ModelLock = { + checkedAt: "2026-11-17", + engines: { + [ENGINE]: { clients: {}, providers: {} }, + // A lock entry with no harness row has nothing to start, so it is not run. + "agent-gone": { clients: {}, providers: {} }, + }, + }; + expect( + probedHarnesses(known, published, lock).map((harness) => harness.image), + ).toEqual(["agent-claude-sdk", "agent-langgraph-agui", ENGINE]); + expect( + probedHarnesses(known, published, lock, [ENGINE]).map( + (harness) => harness.image, + ), + ).toEqual([ENGINE]); +}); + +test("langgraph-ts is run on Anthropic, OpenAI and Gemini, one row per listed model", () => { + expect(providersFor(LANGGRAPH_TS)).toEqual( + expect.arrayContaining(["anthropic", "openai", "google"]), + ); + const rows = buildMatrix({ + harnesses: [LANGGRAPH_TS], + tags: ["source"], + listings: new Map([ + ["anthropic", { models: anthropicIds }], + ["openai", { models: openaiIds }], + ["google", { models: geminiIds }], + ]), + }); + const runs = rows.filter((matrixRow) => matrixRow.kind === "run"); + expect(runs.length).toBe( + anthropicIds.length + openaiIds.length + geminiIds.length, + ); + for (const [provider, ids] of [ + ["anthropic", anthropicIds], + ["openai", openaiIds], + ["google", geminiIds], + ] as const) + for (const model of ids) + expect( + runs.some( + (run) => + run.kind === "run" && + run.provider === provider && + run.model === model, + ), + `${provider} ${model}`, + ).toBe(true); +}); + +test("the probe reads an engine's LangChain and AG-UI clients under plain Node", async () => { + // A Node 22 image's working directory, with its packages installed beside it (and one hoisted + // to the directory above, as a workspace install leaves it). + const image = mkdtempSync(join(tmpdir(), "model-compat-node-")); + const app = join(image, "app"); + const installed: Record = { + "@langchain/langgraph": "1.7.5", + "@langchain/core": "1.2.0", + "@langchain/anthropic": "1.3.1", + "@langchain/openai": "1.4.0", + "@langchain/google-genai": "2.1.0", + "@ag-ui/client": "0.0.46", + }; + for (const [name, version] of Object.entries(installed)) { + const dir = join(app, "node_modules", name); + mkdirSync(dir, { recursive: true }); + // An `exports` map without `./package.json`, as some packages ship: a plain require of the + // package's manifest is refused, so the probe reads the file. + writeFileSync( + join(dir, "package.json"), + JSON.stringify({ name, version, exports: { ".": "./index.js" } }), + ); + } + const hoisted = join(image, "node_modules", "@ag-ui", "core"); + mkdirSync(hoisted, { recursive: true }); + writeFileSync( + join(hoisted, "package.json"), + JSON.stringify({ name: "@ag-ui/core", version: "0.0.46" }), + ); + + const child = Bun.spawn(["node", "-e", NODE_VERSIONS], { + cwd: app, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, code] = await Promise.all([ + new Response(child.stdout).text(), + child.exited, + ]); + expect(code).toBe(0); + expect(JSON.parse(stdout.trim())).toEqual({ + ...installed, + "@ag-ui/core": "0.0.46", + }); +}); + +test("both workflows check every engine the lock names, not only the published ones", () => { + for (const workflow of ["model-compat.yml", "model-lock.yml"]) { + const text = readFileSync( + join(ROOT, ".github/workflows", workflow), + "utf8", + ); + expect(text, workflow).toContain("bun scripts/model-compat.ts harnesses"); + } +}); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index 2fa35dbdb7..e099dedcb7 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -35,6 +35,12 @@ * bun scripts/model-compat.ts report --summary [--issue ] results/*.json * bun scripts/model-compat.ts lock [--lock shared/model-lock.json] [--date YYYY-MM-DD] results/*.json * bun scripts/model-compat.ts guard [--lock shared/model-lock.json] [--summary ] results/*.json + * bun scripts/model-compat.ts harnesses + * + * The harnesses it runs are the published ones and every engine the lock names, published or not + * yet (`harnesses` prints them, for the workflows' matrix). An engine is put in the lock with its + * providers' pairs empty before its first release, so nothing is offered on it until it has + * passed here: the LangGraph (TypeScript) engine, agent-langgraph-ts, starts that way. * * Needs Docker. Exits 1 when anything failed (unless --exit-zero); `report` merges the results of * several runs, writes the Markdown summary, and exits 1 when any of them failed. `lock` exits 0, @@ -347,6 +353,43 @@ export function providersFor(harness: Harness): ProviderId[] { } } +/** + * The harnesses the check runs: every published harness, and every engine the model lock names + * whether it is published yet or not. An engine the lock names offers a model only once it has + * passed here, so it is checked from before its first release (its `source` image, built from this + * checkout; a `latest` that isn't published yet is an inconclusive row). A lock entry with no row + * in shared/harnesses.json has nothing to start, so it is not run. `only` narrows to the named + * image directories. + */ +export function probedHarnesses( + known: readonly Harness[], + published: readonly string[], + lock: ModelLock | undefined, + only?: readonly string[], +): Harness[] { + const locked = Object.keys(lock?.engines ?? {}); + return known.filter( + (row) => + !!row.image && + (published.includes(row.image) || locked.includes(row.image)) && + (!only || only.includes(row.image)), + ); +} + +function readProbed(only?: readonly string[]): Harness[] { + const published: string[] = JSON.parse( + readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), + ); + const { harnesses } = JSON.parse( + readFileSync(join(ROOT, "shared/harnesses.json"), "utf8"), + ) as { harnesses: Harness[] }; + const lockPath = join(ROOT, "shared/model-lock.json"); + const lock = existsSync(lockPath) + ? (JSON.parse(readFileSync(lockPath, "utf8")) as ModelLock) + : undefined; + return probedHarnesses(harnesses, published, lock, only); +} + export type Listing = { models: string[] } | { error: string }; export type MatrixRow = @@ -829,6 +872,17 @@ export const NODE_CLIENTS = [ "@ai-sdk/openai", "@ai-sdk/anthropic", "@ai-sdk/google", + // The LangGraph (TypeScript) engine: its graph, the LangChain chat models that call each + // provider, and the AG-UI packages it speaks through. + "@langchain/langgraph", + "@langchain/core", + "@langchain/anthropic", + "@langchain/openai", + "@langchain/google-genai", + "@ag-ui/core", + "@ag-ui/client", + "@ag-ui/encoder", + "@ag-ui/langgraph", ]; const PYTHON_VERSIONS = ` @@ -850,10 +904,23 @@ except Exception: pass print(json.dumps(found)) `; -const NODE_VERSIONS = ` +/** + * Read under `bun -e` or plain `node -e` (a Node image may have no Bun), from the image's working + * directory up, the way Node resolves a package. The manifest is read as a file rather than + * required, since a package whose `exports` map leaves out `./package.json` refuses the require. + */ +export const NODE_VERSIONS = ` +const { readFileSync } = require("node:fs"); +const { dirname, join } = require("node:path"); const found = {}; for (const name of ${JSON.stringify(NODE_CLIENTS)}) { - try { found[name] = require(\`\${process.cwd()}/node_modules/\${name}/package.json\`).version; } catch {} + for (let dir = process.cwd(); ; dir = dirname(dir)) { + try { + found[name] = JSON.parse(readFileSync(join(dir, "node_modules", name, "package.json"), "utf8")).version; + break; + } catch {} + if (dirname(dir) === dir) break; + } } console.log(JSON.stringify(found)); `; @@ -863,6 +930,7 @@ async function clientsIn(image: string): Promise> { for (const args of [ ["python", "-c", PYTHON_VERSIONS], ["bun", "-e", NODE_VERSIONS], + ["node", "-e", NODE_VERSIONS], ] as const) { const ran = await exec([ "docker", @@ -1043,21 +1111,11 @@ async function check(argv: string[]): Promise { const keys = readKeys(options.envFile); const scrub = scrubber(keys); - const published: string[] = JSON.parse( - readFileSync(join(ROOT, ".github/published-images.json"), "utf8"), - ); - const harnesses = ( - JSON.parse(readFileSync(join(ROOT, "shared/harnesses.json"), "utf8")) as { - harnesses: Harness[]; - } - ).harnesses.filter( - (row) => - row.image && - published.includes(row.image) && - (!options.harnesses || options.harnesses.includes(row.image)), - ); + const harnesses = readProbed(options.harnesses); if (!harnesses.length) - throw new Error("No published harness matches --harness."); + throw new Error( + "No harness matches --harness: it must be published, or named in shared/model-lock.json, and have a row in shared/harnesses.json.", + ); const wanted = new Set( harnesses @@ -1621,6 +1679,13 @@ function report(argv: string[]): number { if (import.meta.main) { const argv = process.argv.slice(2); + if (argv[0] === "harnesses") { + // The workflows' matrix: one job per harness this check runs, as compact JSON. + console.log( + JSON.stringify(readProbed().map((harness) => harness.image ?? "")), + ); + process.exit(0); + } const code = argv[0] === "report" ? report(argv.slice(1)) diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts index 52d96f72cc..a9e8263f43 100644 --- a/shared/model-lock.test.ts +++ b/shared/model-lock.test.ts @@ -129,6 +129,14 @@ test("every engine the lock names is a harness OpenBot publishes, with the clien const images = harnesses.map((harness) => harness.image); for (const [engine, entry] of Object.entries(MODEL_LOCK.engines)) { expect(images, engine).toContain(engine); - expect(Object.keys(entry.clients).length, engine).toBeGreaterThan(0); + // An engine locked before its first run (agent-langgraph-ts) has run no client yet, and + // offers nothing until it has. + const ran = Object.values(entry.providers).some( + (pair) => + !!pair && + (pair.verified.length > 0 || Object.keys(pair.refused).length > 0), + ); + if (ran) + expect(Object.keys(entry.clients).length, engine).toBeGreaterThan(0); } }); From f873844c48a954097367dd1bdba392d2534cbbd7 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 15:28:33 -0800 Subject: [PATCH 13/18] Model check: read the LangGraph (TypeScript) engine's Gemini client, and test the lock rule on its own lock (#1793) The engine ships @langchain/google, which the probe did not read, so a run recorded no Gemini client for it. Two tests read the live model lock and could only pass while it was empty, so the first lock refresh after a real run would have failed them; they now build on an unrun lock of their own. --- scripts/model-compat-langgraph-ts.test.ts | 20 ++++++++++++++++++-- scripts/model-compat.ts | 1 + 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/model-compat-langgraph-ts.test.ts b/scripts/model-compat-langgraph-ts.test.ts index 0f84162835..1f0f15b5f2 100644 --- a/scripts/model-compat-langgraph-ts.test.ts +++ b/scripts/model-compat-langgraph-ts.test.ts @@ -46,6 +46,20 @@ const anthropicIds = usableModels( const openaiIds = usableModels("openai", openaiListing, new Map(), NOW); const geminiIds = ["gemini-3.8-flash", "gemini-4-argon", "gemini-3.5-pro"]; +const UNRUN: ModelLock = { + checkedAt: "2026-11-16", + engines: { + [ENGINE]: { + clients: {}, + providers: { + anthropic: { verified: [], refused: {} }, + openai: { verified: [], refused: {} }, + google: { verified: [], refused: {} }, + }, + }, + }, +}; + test("before its first run, nothing is offered on langgraph-ts, on any provider", () => { expect(anthropicIds.length).toBeGreaterThan(0); expect(openaiIds.length).toBeGreaterThan(0); @@ -56,7 +70,7 @@ test("before its first run, nothing is offered on langgraph-ts, on any provider" ] as const) for (const model of ids) expect( - lockAllows(MODEL_LOCK, ENGINE, provider, model), + lockAllows(UNRUN, ENGINE, provider, model), `${provider} ${model}`, ).toBe(false); }); @@ -125,7 +139,7 @@ test("after a run, exactly the models that answered are offered on langgraph-ts" }, }, }; - const lock = buildLock(MODEL_LOCK, [run], "2026-11-17"); + const lock = buildLock(UNRUN, [run], "2026-11-17"); const allowed = ( provider: "anthropic" | "openai" | "google", ids: string[], @@ -289,6 +303,8 @@ test("the probe reads an engine's LangChain and AG-UI clients under plain Node", "@langchain/anthropic": "1.3.1", "@langchain/openai": "1.4.0", "@langchain/google-genai": "2.1.0", + // The engine's Gemini client, the one its package.json names. + "@langchain/google": "0.2.15", "@ag-ui/client": "0.0.46", }; for (const [name, version] of Object.entries(installed)) { diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index e099dedcb7..d84bc5b2fc 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -878,6 +878,7 @@ export const NODE_CLIENTS = [ "@langchain/core", "@langchain/anthropic", "@langchain/openai", + "@langchain/google", "@langchain/google-genai", "@ag-ui/core", "@ag-ui/client", From 70ec177d92208d017120b97cac75f36d4d125fec Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 15:30:32 -0800 Subject: [PATCH 14/18] Lock the LangGraph (TypeScript) engine from its probe run, and never leave a default engine empty agent-langgraph-ts's entry comes from its own run: Anthropic 15, OpenAI 29 and Google 18 verified. The two Gemma 4 models are refused: they answer with their reasoning in the reply text. A test now fails when an engine new Bots run on by default verifies nothing on a provider it is locked for, which the empty pairs it was first locked with would have done to every new Bot. --- docs/releasing.md | 7 ++- scripts/model-lock.test.ts | 19 +++++++- shared/model-lock.json | 97 ++++++++++++++++++++++++++++++++++++++ shared/model-lock.test.ts | 4 +- shared/model-lock.ts | 15 ++++++ 5 files changed, 138 insertions(+), 4 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index 2c37aed788..990b386615 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1141,8 +1141,11 @@ the harness is hidden, and a conversation already set to one runs on the default - **A new engine** can be locked before its first release, by giving it empty pairs for the providers it runs on. Nothing is offered on it until a run proves a model, including on a provider the check has no key for. The check runs every engine the lock names, published or not - yet, from its `source` image (`bun scripts/model-compat.ts harnesses` lists them). The LangGraph - (TypeScript) engine, `agent-langgraph-ts`, starts this way on Anthropic, OpenAI and Google. + yet, from its `source` image (`bun scripts/model-compat.ts harnesses` lists them). An engine new + Bots run on by default (`DEFAULT_ENGINES` in `shared/model-lock.ts`, today the LangGraph + (TypeScript) engine, `agent-langgraph-ts`) must verify at least one model on every provider it is + locked for, or `scripts/model-lock.test.ts` fails: write its entry from a run, never empty. The + lock can't reach main before an engine it names has its row in `shared/harnesses.json`. - **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that started and stopped answering and the clients that moved. Merging it lets the picker offer them. diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index e796077e72..e294e67814 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -1,7 +1,12 @@ import { expect, test } from "bun:test"; import { readdirSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { MODEL_LOCK, type ModelLock } from "../shared/model-lock"; +import { + DEFAULT_ENGINES, + emptyPairs, + MODEL_LOCK, + type ModelLock, +} from "../shared/model-lock"; import { brokenVerified, buildLock, @@ -28,6 +33,18 @@ test("the lock was proven on the model clients each harness pins in this commit" expect(staleClients(MODEL_LOCK, ROOT)).toEqual([]); }); +/* + * A DEFAULT ENGINE ALWAYS HAS SOMETHING TO OFFER. The LangGraph (TypeScript) engine was first + * locked with its pairs empty, so nothing would be offered on it before a run proved a model: had + * that reached main before a run, every new Bot on it would have had no model at all. + */ +test("every engine new Bots run on by default verifies a model on each provider it is locked for", () => { + for (const engine of DEFAULT_ENGINES) { + expect(MODEL_LOCK.engines[engine], engine).toBeDefined(); + expect(emptyPairs(MODEL_LOCK, engine), engine).toEqual([]); + } +}); + test("every exact pin in a harness's requirements is a client the check records", () => { const harnesses = readdirSync(ROOT).filter((name) => name.startsWith("agent-"), diff --git a/shared/model-lock.json b/shared/model-lock.json index 0b1b2174ee..f3f6657241 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -547,6 +547,103 @@ } } }, + "agent-langgraph-ts": { + "clients": { + "@ag-ui/client": "1.0.5", + "@ag-ui/core": "1.0.5", + "@ag-ui/encoder": "1.0.5", + "@langchain/anthropic": "1.6.5", + "@langchain/core": "1.4.1", + "@langchain/langgraph": "1.7.5", + "@langchain/openai": "1.8.4" + }, + "providers": { + "anthropic": { + "verified": [ + "claude-fable-5", + "claude-fable-5-1", + "claude-fable-5-2", + "claude-haiku-4-5-20251001", + "claude-haiku-5-5", + "claude-opus-4-5-20251101", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-opus-5", + "claude-opus-5-5", + "claude-sonnet-4-5-20250929", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-sonnet-5-5" + ], + "refused": {} + }, + "google": { + "verified": [ + "gemini-2.5-flash", + "gemini-2.5-pro", + "gemini-3-flash-preview", + "gemini-3.1-flash-lite", + "gemini-3.1-flash-lite-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-pro-preview-customtools", + "gemini-3.5-flash", + "gemini-3.5-flash-lite", + "gemini-3.6-flash", + "gemini-3.7-flash", + "gemini-3.8-flash", + "gemini-4-argon", + "gemini-4-carbon", + "gemini-4-hydrogen", + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest" + ], + "refused": { + "gemini-2.5-flash-lite": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/...", + "gemini-decision-1": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/javascript/langchain/errors/MODEL_NOT_FOUND/", + "gemini-omni-1.1-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", + "gemini-omni-1.2-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", + "gemma-4-26b-a4b-it": "answers with its reasoning (...) in the reply text, which a person would read as the Bot's answer", + "gemma-4-31b-it": "answers with its reasoning (...) in the reply text, which a person would read as the Bot's answer" + } + }, + "openai": { + "verified": [ + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + "gpt-6-astra", + "gpt-6-luna", + "gpt-6-sol", + "gpt-6.1-sol", + "gpt-6.1-terra", + "gpt-6.2-astra", + "gpt-6.2-sol", + "o3", + "o3-pro" + ], + "refused": {} + } + } + }, "agent-langroid": { "clients": { "google-genai": "3.4.0", diff --git a/shared/model-lock.test.ts b/shared/model-lock.test.ts index a9e8263f43..de8d9bec45 100644 --- a/shared/model-lock.test.ts +++ b/shared/model-lock.test.ts @@ -122,7 +122,9 @@ test("what the check never ran is open, so nobody loses their models to missing ).toBe(true); }); -test("every engine the lock names is a harness OpenBot publishes, with the clients it ran", async () => { +// Also why the lock can't reach main before an engine it names: agent-langgraph-ts needs its row in +// shared/harnesses.json first. +test("every engine the lock names is a harness OpenBot knows, with the clients it ran", async () => { const { harnesses } = (await import("./harnesses.json")).default as { harnesses: { image: string | null }[]; }; diff --git a/shared/model-lock.ts b/shared/model-lock.ts index 9cb547f221..1346132372 100644 --- a/shared/model-lock.ts +++ b/shared/model-lock.ts @@ -52,6 +52,21 @@ export type ModelLock = { export const MODEL_LOCK: ModelLock = manifest as ModelLock; +/** + * The engines new Bots run on unless somebody picks another. A locked pair on one of these with + * nothing verified would leave every new Bot with no model to offer on that provider, so a test + * holds each to at least one (scripts/model-lock.test.ts). + */ +export const DEFAULT_ENGINES: readonly string[] = ["agent-langgraph-ts"]; + +/** The locked pairs of an engine that verify nothing: no model would be offered on them. */ +export function emptyPairs(lock: ModelLock, engine: string): LockProvider[] { + const providers = lock.engines[engine]?.providers ?? {}; + return (Object.keys(providers) as LockProvider[]).filter( + (provider) => (providers[provider]?.verified.length ?? 0) === 0, + ); +} + /** * What the lock says about one model on one harness and provider. * From d3d64c0e3cc837813ede16dcb885e9da88709770 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 20:55:54 -0800 Subject: [PATCH 15/18] Hold the LangGraph (TypeScript) engine's npm lockfile to the lock, and fail a reply that carries its reasoning The staleness guard now reads package-lock.json too, which showed the engine's entry was written before its Gemini client was recorded. A reply with the model's reasoning in it (Gemma 4's through this engine) now fails the check in its own words, so the lock refuses it on every run instead of by hand. --- scripts/model-compat.test.ts | 11 +++++++++++ scripts/model-compat.ts | 27 ++++++++++++++++++++++++++- scripts/model-lock.test.ts | 4 ++++ 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/scripts/model-compat.test.ts b/scripts/model-compat.test.ts index c4ce089879..9fcc62e67e 100644 --- a/scripts/model-compat.test.ts +++ b/scripts/model-compat.test.ts @@ -233,6 +233,17 @@ test("a run that answered passes", () => { }); }); +test("a reply that carries the model's reasoning fails, though it says OK", () => { + // What gemma-4-31b-it answered through the LangGraph (TypeScript) engine. + const leaked = agnoReply.replace( + '"delta":"OK"', + '"delta":"The user wants one word.OK"', + ); + const outcome = classifyRun({ status: 200, body: leaked }); + expect(outcome.status).toBe("fail"); + expect(outcome.detail).toContain("reasoning in the reply text"); +}); + test("an error a framework puts in its raw event is still the failure's text", () => { const result = classifyRun({ status: 200, body: agnoNotFound }); expect(result.status).toBe("fail"); diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index d84bc5b2fc..351691ec7d 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -512,7 +512,18 @@ export function classifyRun(response: { ); } -function judgeReply(text: string, silence: string): Outcome { +/** + * A model's own reasoning left in its reply, which a person would read as the Bot's answer: the + * Gemma 4 models, through the LangGraph (TypeScript) engine, answered "…OK". + */ +const REASONING_IN_REPLY = /<\/?(thought|think|thinking|reasoning)>/i; + +export function judgeReply(text: string, silence: string): Outcome { + if (REASONING_IN_REPLY.test(text)) + return { + status: "fail", + detail: `answered with its reasoning in the reply text: ${oneLine(text, 160)}`, + }; if (/\bok(ay)?\b/i.test(text)) return { status: "pass", detail: oneLine(text, 80) }; if (text) @@ -1519,6 +1530,20 @@ export function pinnedClients( )?.[1]; if (found) pinned[name] = found; } + // npm's lockfile, for a harness installed with `npm ci` (the LangGraph (TypeScript) engine). + const npmLock = read("package-lock.json"); + if (npmLock) { + const packages = + ( + JSON.parse(npmLock) as { + packages?: Record; + } + ).packages ?? {}; + for (const name of NODE_CLIENTS) { + const version = packages[`node_modules/${name}`]?.version; + if (version) pinned[name] = version; + } + } const codex = /^DEFAULT_CLIENT_VERSION\s*=\s*"([^"]+)"/m.exec( read("src/chatgpt_models.py"), )?.[1]; diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index e294e67814..e5d8e997c7 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -92,6 +92,10 @@ test("a pin that moved past the lock is caught", () => { expect(Object.keys(pinnedClients(ROOT, "agent-mastra"))).toContain( "@mastra/core", ); + // npm's lockfile, for the LangGraph (TypeScript) engine. + expect(Object.keys(pinnedClients(ROOT, "agent-langgraph-ts"))).toContain( + "@langchain/core", + ); }); /* From 046bc430f6be62d9399ee4edaec75f9806073474 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 21:18:07 -0800 Subject: [PATCH 16/18] Write the model lock from this branch's run on main, LangGraph (TypeScript) included From the Model lock run on d3d64c0e3 with the repository's keys, every engine's source image on main's pins. agent-langgraph-ts verifies Anthropic 15, OpenAI 29 and Google 18, with its Gemini client recorded. The Gemma 4 models are refused on every engine that reaches Google through its OpenAI-compatible API: they answer with their reasoning in the reply. --- shared/model-lock.json | 98 +++++++++++++++++++++--------------------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/shared/model-lock.json b/shared/model-lock.json index f3f6657241..a626cea5cf 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -1,5 +1,5 @@ { - "checkedAt": "2026-11-17", + "checkedAt": "2026-11-18", "engines": { "agent-adk": { "clients": { @@ -48,13 +48,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with the instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should check if there are any constraints or tool calls needed. No tool calls are needed. The re..." } }, "openai": { @@ -97,7 +97,7 @@ "clients": { "ag2": "1.3.2", "anthropic": "2.10.0", - "openai": "3.53.0" + "openai": "3.54.0" }, "providers": { "anthropic": { @@ -139,13 +139,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "BadRequestError('Error code: 400')", - "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')" + "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user is asking me to reply with the single word \"OK\". This is a direct instruction. I should comply.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". 1. **Analyze the user's request:** The user explicitly asked for the single word \"OK\". 2. **Check ..." } }, "openai": { @@ -278,7 +278,7 @@ }, "agent-antigravity": { "clients": { - "google-genai": "3.4.0" + "google-genai": "3.5.0" }, "providers": { "google": { @@ -300,8 +300,7 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", @@ -386,15 +385,15 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { - "gemini-2.5-flash-lite": "thread=model-compat-21c5bc66a9c4 run=run-21c5bc66a9c4: CrewAI flow failed; see server logs", - "gemini-decision-1": "thread=model-compat-582497527ff7 run=run-582497527ff7: CrewAI flow failed; see server logs", - "gemini-omni-1.1-flash": "thread=model-compat-ce043e6f69ce run=run-ce043e6f69ce: CrewAI flow failed; see server logs", - "gemini-omni-1.2-flash": "thread=model-compat-702a8e9c82e6 run=run-702a8e9c82e6: CrewAI flow failed; see server logs" + "gemini-2.5-flash-lite": "thread=model-compat-e3ae8f99404b run=run-e3ae8f99404b: CrewAI flow failed; see server logs", + "gemini-decision-1": "thread=model-compat-103e0f83ec4c run=run-103e0f83ec4c: CrewAI flow failed; see server logs", + "gemini-omni-1.1-flash": "thread=model-compat-d22bbf9f4055 run=run-d22bbf9f4055: CrewAI flow failed; see server logs", + "gemini-omni-1.2-flash": "thread=model-compat-3acce5a2103b run=run-3acce5a2103b: CrewAI flow failed; see server logs", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -440,11 +439,11 @@ "codex-client": "0.167.0", "google-genai": "2.34.0", "langchain-anthropic": "1.7.10", - "langchain-core": "1.8.1", + "langchain-core": "1.8.2", "langchain-google-genai": "4.4.3", "langchain-openai": "1.8.1", "langgraph": "1.2.22", - "openai": "3.53.0" + "openai": "3.54.0" }, "providers": { "anthropic": { @@ -502,13 +501,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "Error code: 400", - "gemini-omni-1.2-flash": "Error code: 400" + "gemini-omni-1.2-flash": "Error code: 400", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single word response: \"OK\". I should comply with the instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -554,6 +553,7 @@ "@ag-ui/encoder": "1.0.5", "@langchain/anthropic": "1.6.5", "@langchain/core": "1.4.1", + "@langchain/google": "0.2.15", "@langchain/langgraph": "1.7.5", "@langchain/openai": "1.8.4" }, @@ -600,12 +600,10 @@ "gemini-pro-latest" ], "refused": { - "gemini-2.5-flash-lite": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/...", - "gemini-decision-1": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"models/gemini-decision-1 is not found for API version v1main, or is not supported for generateContent. Call ModelService.ListModels to see the list of available models and their supported methods.\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/javascript/langchain/errors/MODEL_NOT_FOUND/", "gemini-omni-1.1-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", "gemini-omni-1.2-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", - "gemma-4-26b-a4b-it": "answers with its reasoning (...) in the reply text, which a person would read as the Bot's answer", - "gemma-4-31b-it": "answers with its reasoning (...) in the reply text, which a person would read as the Bot's answer" + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -646,7 +644,7 @@ }, "agent-langroid": { "clients": { - "google-genai": "3.4.0", + "google-genai": "3.5.0", "langroid": "0.69.24", "litellm": "1.105.7", "openai": "2.54.0" @@ -691,13 +689,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -782,13 +780,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -873,15 +871,15 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-2.5-flash-lite": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", "gemini-decision-1": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", "gemini-omni-1.1-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", - "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd..." + "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: * User's instruction: \"Reply with the single word OK.\" * Constraint: Reply with the single word OK. * The user wants a specific single-word response. *...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". The system instructions state: \"Answer the question you are asked, briefly and correctly.\" The us..." } }, "openai": { @@ -924,7 +922,7 @@ "clients": { "agent-framework-core": "1.26.0", "anthropic": "1.17.0", - "openai": "3.53.0" + "openai": "3.54.0" }, "providers": { "anthropic": { @@ -1016,8 +1014,8 @@ "clients": { "anthropic": "1.17.0", "google-genai": "2.34.0", - "openai": "3.53.0", - "pydantic-ai-slim": "2.69.0" + "openai": "3.54.0", + "pydantic-ai-slim": "2.70.0" }, "providers": { "anthropic": { @@ -1151,13 +1149,13 @@ "gemini-4-hydrogen", "gemini-flash-latest", "gemini-flash-lite-latest", - "gemini-pro-latest", - "gemma-4-26b-a4b-it", - "gemma-4-31b-it" + "gemini-pro-latest" ], "refused": { "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]" + "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with only the word \"OK\". 1. **Analyze the user prompt:** \"Reply with the single word OK.\" 2. **Identify the constraint:** Th...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a response consisting of only the single word \"OK\". No tools are needed. The request is straightforward and doesn't violate any safety g..." } }, "openai": { From d6e4e12db66b5ed554ba74e95c4e4f342c9bfe5d Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 22:40:37 -0800 Subject: [PATCH 17/18] Refuse a model the provider retires, and ask a failing model once more before it counts Google now answers gemini-2.5-flash-lite with "no longer available to new users": the lock refuses it, so a person set to it is told, and it never fails a pull request since every image fails it. A one-off timeout (gemini-4-carbon through Antigravity's source image, which answered through the published image in the same run) no longer counts: every failing run is asked once more. --- docs/releasing.md | 7 ++++--- scripts/model-compat.ts | 42 ++++++++++++++++++++++++++++++-------- scripts/model-lock.test.ts | 40 +++++++++++++++++++++++++++++------- 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/docs/releasing.md b/docs/releasing.md index 990b386615..7fbfd16356 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -1149,9 +1149,10 @@ the harness is hidden, and a conversation already set to one runs on the default - **A new model** stays hidden until a daily run proves it. The run then opens or updates one pull request, **Refresh the model lock**, on the branch `model-lock/refresh`, listing the models that started and stopped answering and the clients that moved. Merging it lets the picker offer them. - A failure that says nothing about the client keeps what the lock said before: the harness never - started, a timeout, a rate limit, or a model the check's key can't use (Google closes older models - to new projects, so two keys can disagree about the same model through the same client). + Every failure is asked once more before it counts. A failure that says nothing about the client + keeps what the lock said before: the harness never started, a timeout, a rate limit, or a bare + 404 for a model the check's key can't see. A model the provider retires ("no longer available to + new users") is refused, so a person set to it is told, and it never fails a pull request. - **A client bump** is held to the lock: a pull request from this repository that changes a harness's `requirements.txt`, `package.json`, `bun.lock`, `Dockerfile` or the Codex client version runs the **Model lock** workflow, which builds that harness, sends every listed model a request diff --git a/scripts/model-compat.ts b/scripts/model-compat.ts index 351691ec7d..2a0b97742c 100644 --- a/scripts/model-compat.ts +++ b/scripts/model-compat.ts @@ -1272,12 +1272,21 @@ async function check(argv: string[]): Promise { let done = 0; await pool(runs, options.concurrency, async (row) => { const image = imageOf(row.harness, row.tag); - const outcome = unpulled.has(image) + const ask = () => runOne(row, keys.get(row.provider) ?? "", scrub); + /* + * A failure is asked once more before it counts. A provider's refusal says the same thing + * twice; a run that timed out once does not (gemini-4-carbon through Antigravity's source + * image on 2026-11-18, which answered through the published image in the same run and through + * every other engine), and must not fail a pull request or leave the lock. + */ + let outcome: Outcome = unpulled.has(image) ? { status: "fail" as const, detail: `could not get ${image}: ${unpulled.get(image)}`, } - : await runOne(row, keys.get(row.provider) ?? "", scrub); + : await ask(); + if (outcome.status === "fail" && !unpulled.has(image)) + outcome = await ask(); results.push({ harness: row.harness.image ?? "", tag: row.tag, @@ -1341,14 +1350,24 @@ const RATE_LIMITED = /\b429\b|rate.?limit|RESOURCE_EXHAUSTED|too many requests/i; /** - * A model this key's account can't use, which another account can: Google closes older models to - * new projects ("no longer available to new users") and answers a model a project can't see with a - * bare 404. Seen on 2026-11-17, when two Gemini keys disagreed on gemini-2.5-pro and - * gemini-2.5-flash-lite through the same clients. About the key, not the client. A 404 that names - * the client's endpoint ("only supported in v1/responses") is the client's, and stays a refusal. + * A model this key's account can't see, which another account may: a bare 404, or one that names + * no reason. About the key, not the client. A 404 that names the client's endpoint ("only supported + * in v1/responses") is the client's, and stays a refusal. */ const UNAVAILABLE_TO_KEY = - /no longer available to new users|is not found for API version|does not exist or you do not have access|do not have access to (?:the )?model|^(?:NotFoundError\(')?Error code: 404'?\)?$|^Not Found$/i; + /is not found for API version|does not exist or you do not have access|do not have access to (?:the )?model|^(?:NotFoundError\(')?Error code: 404'?\)?$|^Not Found$/i; + +/** + * A provider retiring a model: Google's "no longer available to new users, use …". The check + * working as designed: the lock refuses it, so a person set to it is told and moved to the default + * instead of meeting the provider's 404. Not a client's fault, so it never fails a pull request. + */ +const RETIRED = + /no longer available to new users|has been (?:deprecated|retired|shut down)|(?:was|is) (?:deprecated|retired) and (?:is )?no longer/i; + +export function retiredByProvider(detail: string): boolean { + return RETIRED.test(detail); +} /** A failure about the check's key, not the harness's client: a rate limit, or a model it can't see. */ export function aboutTheKey(detail: string): boolean { @@ -1601,7 +1620,12 @@ export function brokenVerified( if (row.status !== "fail" || row.model === "(all)") return false; if (row.tag !== SOURCE) return false; // The key's account, not the change's client: says nothing about the change. - if (aboutTheKey(row.detail) || failsOnPublished(row)) return false; + if ( + aboutTheKey(row.detail) || + retiredByProvider(row.detail) || + failsOnPublished(row) + ) + return false; const id = providerIdOf(row); return ( !!id && diff --git a/scripts/model-lock.test.ts b/scripts/model-lock.test.ts index e5d8e997c7..dec044c612 100644 --- a/scripts/model-lock.test.ts +++ b/scripts/model-lock.test.ts @@ -325,6 +325,36 @@ test("a client bump that breaks a verified model is caught, and one the lock alr ); }); +test("a model the provider retires is refused, and fails no pull request", () => { + // What Google answered through Antigravity on 2026-11-18, for a model the lock had verified. + const RETIRED_SAID = + 'AntigravityExecutionError: Agent execution terminated due to error. ("agent executor error: Error 404, Message: This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements.'; + const google = (status: Result["status"], detail = "OK", tag = "source") => + row("gemini-2.5-flash-lite", status, detail, { + harness: "agent-antigravity", + provider: "Google", + providerId: "google", + tag, + }); + const before = buildLock( + undefined, + [{ results: [google("pass")] }], + "2026-11-17", + ); + const after = buildLock( + before, + [{ results: [google("fail", RETIRED_SAID)] }], + "2026-11-18", + ); + const pair = after.engines["agent-antigravity"]?.providers.google; + expect(pair?.verified).toEqual([]); + expect(pair?.refused["gemini-2.5-flash-lite"]).toContain( + "no longer available to new users", + ); + // The provider's doing, on every image: not the change's. + expect(brokenVerified(before, [google("fail", RETIRED_SAID)])).toEqual([]); +}); + test("a model the check's key can't use is the key's, not the client's", () => { const google = (model: string, status: Result["status"], detail = "OK") => row(model, status, detail, { @@ -332,18 +362,14 @@ test("a model the check's key can't use is the key's, not the client's", () => { provider: "Google", providerId: "google", }); - // One Gemini key's run on 2026-11-17; another key, through the same image, disagreed on both. + // One Gemini key's run on 2026-11-17; another key, through the same image, disagreed. const lock = buildLock( undefined, [ { results: [ google("gemini-2.5-flash-lite", "pass"), - google( - "gemini-2.5-pro", - "fail", - "Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-pro is no longer available to new users.'}}]", - ), + google("gemini-decision-1", "fail", "Error code: 404"), google( "gemini-omni-1.1-flash", "fail", @@ -356,7 +382,7 @@ test("a model the check's key can't use is the key's, not the client's", () => { ); const pair = lock.engines["agent-langgraph-agui"]?.providers.google; expect(pair?.verified).toEqual(["gemini-2.5-flash-lite"]); - // Not refused: another key may run it, and a later run with one proves it. + // A bare 404 isn't refused: another key may run it, and a later run with one proves it. expect(Object.keys(pair?.refused ?? {})).toEqual(["gemini-omni-1.1-flash"]); expect( brokenVerified(lock, [ From 9bc8280d8812737269afbb8dbc3ef5d6c6c579c5 Mon Sep 17 00:00:00 2001 From: David McKay Date: Tue, 17 Nov 2026 22:53:01 -0800 Subject: [PATCH 18/18] Write the model lock from this branch's run: gemini-2.5-flash-lite is retired From the Model lock run on d6e4e12db with the repository's keys. Google now answers gemini-2.5-flash-lite "no longer available to new users", so it is refused where the engine passes Google's words on, and unverified where it doesn't. Every other pair is unchanged. --- shared/model-lock.json | 55 +++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/shared/model-lock.json b/shared/model-lock.json index a626cea5cf..3437cab84b 100644 --- a/shared/model-lock.json +++ b/shared/model-lock.json @@ -51,10 +51,11 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with the instruction.OK", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should check if there are any constraints or tool calls needed. No tool calls are needed. The re..." + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow the instruction exactly.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" } }, "openai": { @@ -97,7 +98,7 @@ "clients": { "ag2": "1.3.2", "anthropic": "2.10.0", - "openai": "3.54.0" + "openai": "3.55.0" }, "providers": { "anthropic": { @@ -144,8 +145,8 @@ "refused": { "gemini-omni-1.1-flash": "BadRequestError('Error code: 400')", "gemini-omni-1.2-flash": "BadRequestError('Error code: 400')", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user is asking me to reply with the single word \"OK\". This is a direct instruction. I should comply.OK", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". 1. **Analyze the user's request:** The user explicitly asked for the single word \"OK\". 2. **Check ..." + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with the instruction.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user is asking for a specific single-word response: \"OK\". No tool calls are needed. The response should be exactly \"OK\".OK" } }, "openai": { @@ -303,6 +304,7 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 404, Message: This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started)., Status: NOT_FOU...", "gemini-omni-1.1-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)", "gemini-omni-1.2-flash": "AntigravityExecutionError: Agent execution terminated due to error. (\"agent executor error: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []: Error 400, Message: This model only supports Interactions API., Status: INVALID_ARGUMENT, Details: []\") (HTTP 400)" } @@ -313,8 +315,7 @@ "clients": { "ag-ui-claude-sdk": "0.1.5", "anthropic": "2.10.0", - "claude-agent-sdk": "0.2.182", - "claude-code": "2.1.325" + "claude-agent-sdk": "0.2.182" }, "providers": { "anthropic": { @@ -388,12 +389,12 @@ "gemini-pro-latest" ], "refused": { - "gemini-2.5-flash-lite": "thread=model-compat-e3ae8f99404b run=run-e3ae8f99404b: CrewAI flow failed; see server logs", - "gemini-decision-1": "thread=model-compat-103e0f83ec4c run=run-103e0f83ec4c: CrewAI flow failed; see server logs", - "gemini-omni-1.1-flash": "thread=model-compat-d22bbf9f4055 run=run-d22bbf9f4055: CrewAI flow failed; see server logs", - "gemini-omni-1.2-flash": "thread=model-compat-3acce5a2103b run=run-3acce5a2103b: CrewAI flow failed; see server logs", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word \"OK\". I should comply with this instruction.OK", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". I should follow this instruction exactly.OK" + "gemini-2.5-flash-lite": "thread=model-compat-811af2816788 run=run-811af2816788: CrewAI flow failed; see server logs", + "gemini-decision-1": "thread=model-compat-53e7cc2c2ba7 run=run-53e7cc2c2ba7: CrewAI flow failed; see server logs", + "gemini-omni-1.1-flash": "thread=model-compat-bb5958474292 run=run-bb5958474292: CrewAI flow failed; see server logs", + "gemini-omni-1.2-flash": "thread=model-compat-d99e32be6322 run=run-d99e32be6322: CrewAI flow failed; see server logs", + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". This is a direct instruction. I should comply.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". This is a simple constraint following task. No tools are needed.OK" } }, "openai": { @@ -443,7 +444,7 @@ "langchain-google-genai": "4.4.3", "langchain-openai": "1.8.1", "langgraph": "1.2.22", - "openai": "3.54.0" + "openai": "3.55.0" }, "providers": { "anthropic": { @@ -506,8 +507,8 @@ "refused": { "gemini-omni-1.1-flash": "Error code: 400", "gemini-omni-1.2-flash": "Error code: 400", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single word response: \"OK\". I should comply with the instruction.OK", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should follow this instruction exactly.OK" + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". This is a direct instruction to follow a specific format. Plan: 1. Acknowledge the instruction. 2. Output ...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a specific single-word response: \"OK\". I should comply with this request exactly.OK" } }, "openai": { @@ -600,10 +601,11 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "Error code: 404 - [{\"error\":{\"code\":404,\"message\":\"This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).\",\"status\":\"NOT_FOUND\"}}] Troubleshooting URL: https://docs.langchain.com/oss/...", "gemini-omni-1.1-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", "gemini-omni-1.2-flash": "Error code: 400 - [{\"error\":{\"code\":400,\"message\":\"This model only supports Interactions API.\",\"status\":\"INVALID_ARGUMENT\"}}]", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". I should comply with this instruction.OK", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should follow this instruction exactly.OK" + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". This is a direct instruction for a specific output. I should comply.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a single word response: \"OK\".OK" } }, "openai": { @@ -692,6 +694,7 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "Agent error: Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_FOUND'}}]", "gemini-omni-1.1-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "Agent error: Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", @@ -783,6 +786,7 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". I should comply with this instruction.OK", @@ -874,12 +878,12 @@ "gemini-pro-latest" ], "refused": { - "gemini-2.5-flash-lite": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", + "gemini-2.5-flash-lite": "HTTP 500: Not Found", "gemini-decision-1": "HTTP 500: Not Found; logged: stack: \"AI_APICallError: Not Found\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/...", "gemini-omni-1.1-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", "gemini-omni-1.2-flash": "HTTP 500: Bad Request; logged: stack: \"AI_APICallError: Bad Request\\n at (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:4216:15)\\n at async (/app/.mastra/output/node_modules/@ai-sdk/provider-utils/dist/index.js:3618:30)\\n at async (/app/.mastra/output/node_modules/@ai-sd...", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: * User's instruction: \"Reply with the single word OK.\" * Constraint: Reply with the single word OK. * The user wants a specific single-word response. *...", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants me to reply with the single word \"OK\". The system instructions state: \"Answer the question you are asked, briefly and correctly.\" The us..." + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with a single word: \"OK\". * Prompt: \"Reply with the single word OK.\" * Instruction: Answer the question briefly and correctl...", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a response consisting of a single word: \"OK\". Constraint checklist & Confidence score: 1. Single word? Yes. 2. Word is \"OK\"? Yes. Confid..." } }, "openai": { @@ -922,7 +926,7 @@ "clients": { "agent-framework-core": "1.26.0", "anthropic": "1.17.0", - "openai": "3.54.0" + "openai": "3.55.0" }, "providers": { "anthropic": { @@ -1014,7 +1018,7 @@ "clients": { "anthropic": "1.17.0", "google-genai": "2.34.0", - "openai": "3.54.0", + "openai": "3.55.0", "pydantic-ai-slim": "2.70.0" }, "providers": { @@ -1152,10 +1156,11 @@ "gemini-pro-latest" ], "refused": { + "gemini-2.5-flash-lite": "litellm.NotFoundError: NotFoundError: OpenAIException - Error code: 404 - [{'error': {'code': 404, 'message': 'This model models/gemini-2.5-flash-lite is no longer available to new users. Please update your code to use models/gemini-3.5-flash-lite for the latest features and improvements. We recommend you to use the Interactions API (https://aistudio.google.com/docs/get-started).', 'status': 'NOT_...", "gemini-omni-1.1-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", "gemini-omni-1.2-flash": "litellm.BadRequestError: OpenAIException - Error code: 400 - [{'error': {'code': 400, 'message': 'This model only supports Interactions API.', 'status': 'INVALID_ARGUMENT'}}]", - "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants me to reply with only the word \"OK\". 1. **Analyze the user prompt:** \"Reply with the single word OK.\" 2. **Identify the constraint:** Th...", - "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a response consisting of only the single word \"OK\". No tools are needed. The request is straightforward and doesn't violate any safety g..." + "gemma-4-26b-a4b-it": "answered with its reasoning in the reply text: The user wants a single-word response: \"OK\". No tools are needed. Compliance with the constraint is required.OK", + "gemma-4-31b-it": "answered with its reasoning in the reply text: The user wants a specific response: \"OK\". The instruction is very clear and simple. No tool calls are needed.OK" } }, "openai": {